Skip to main content

cpa_server/
request_logging.rs

1//! Go request_logging.go / response_writer.go and request_logger_writer.go.
2//! Request bodies in these files are intentionally unredacted, as in Go. Headers
3//! and URL queries are masked before writing; management routes are never captured.
4
5use cpa_core::gostr::GoStr;
6use std::fs::{File, OpenOptions};
7use std::io::{self, Read, Write};
8use std::path::{Path, PathBuf};
9use std::pin::Pin;
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11use std::sync::{Arc, Mutex, PoisonError};
12use std::task::{Context, Poll};
13
14use axum::body::{Body, HttpBody};
15use axum::extract::{Request, State};
16use axum::http::HeaderMap;
17use axum::middleware::Next;
18use axum::response::Response;
19use bytes::Bytes;
20use chrono::{DateTime, Local, SecondsFormat};
21use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
22
23use crate::Runtime;
24use crate::management::Management;
25use crate::observability::{RequestId, hide_key, mask_query, mask_upstream_query};
26
27const SMALL_BODY: u64 = 1 << 20;
28const DEFERRED_BODY: u64 = 32 << 20;
29static NEXT_ID: AtomicU64 = AtomicU64::new(0);
30
31tokio::task_local! { static CURRENT: RequestLog; }
32
33pub(crate) fn current() -> Option<RequestLog> {
34    CURRENT.try_with(Clone::clone).ok()
35}
36
37#[derive(Clone)]
38struct Captured;
39
40/// Install at the handler boundary, before axum synthesizes Content-Length or
41/// suppresses HEAD bodies. The outer middleware still owns request capture.
42pub(crate) async fn response(request: Request, next: Next) -> Response {
43    let sink = request.extensions().get::<RequestLog>().cloned();
44    let head = request.method() == "HEAD";
45    let cors = request.extensions().get::<crate::management::Cors>().cloned();
46    let mut response = next.run(request).await;
47    if let Some(cors) = &cors {
48        crate::management::cors_headers(&mut response, cors);
49    }
50    match sink {
51        Some(sink) => capture_response(sink, head, response).await,
52        None => response,
53    }
54}
55
56fn setting<'a>(config: &'a cpa_core::config::Config, key: &str) -> Option<&'a serde_yaml_ng::Value> {
57    config.document.get("observability")?.get("logs")?.get(key)
58}
59
60#[derive(Clone)]
61struct CaptureState {
62    rt: Arc<Runtime>,
63    dir: PathBuf,
64}
65
66/// Install inside access logging so the UUID is already in request extensions.
67/// Commercial mode is a construction-time choice, exactly like Go's server.
68pub fn router(management: &Arc<Management>, app: axum::Router) -> axum::Router {
69    if management
70        .rt
71        .config()
72        .document
73        .get("server")
74        .and_then(|s| s.get("commercial-mode"))
75        .and_then(serde_yaml_ng::Value::as_bool)
76        == Some(true)
77    {
78        return app;
79    }
80    // Go resolves relative request-log directories against the config directory.
81    let dir = if management.log_dir.is_absolute() {
82        management.log_dir.clone()
83    } else {
84        management
85            .path
86            .parent()
87            .unwrap_or(Path::new("."))
88            .join(&management.log_dir)
89    };
90    app.layer(axum::middleware::from_fn_with_state(
91        CaptureState {
92            rt: management.rt.clone(),
93            dir,
94        },
95        capture,
96    ))
97}
98
99fn eligible(request: &Request, path: &str) -> bool {
100    if ["/v0/management", "/v8/management", "/management"]
101        .iter()
102        .any(|p| path.starts_with(p))
103    {
104        return false;
105    }
106    request.method() != "GET"
107        || (matches!(path, "/v1/responses" | "/backend-api/codex/responses")
108            && request
109                .headers()
110                .get("upgrade")
111                .is_some_and(|v| trim(v.as_bytes()).eq_ignore_ascii_case(b"websocket")))
112}
113
114async fn capture(State(state): State<CaptureState>, mut request: Request, next: Next) -> Response {
115    let path = crate::management::percent_decode(request.uri().path());
116    if !eligible(&request, &path) {
117        return next.run(request).await;
118    }
119    let config = state.rt.config();
120    let enabled = setting(&config, "request-log").and_then(serde_yaml_ng::Value::as_bool) == Some(true);
121    let multipart = request.headers().get("content-type").is_some_and(|v| {
122        String::from_utf8_lossy(trim(v.as_bytes()))
123            .go_lower()
124            .starts_with("multipart/form-data")
125    });
126    let length = request
127        .headers()
128        .get("content-length")
129        .and_then(|h| h.to_str().ok())
130        .and_then(|s| s.parse::<u64>().ok());
131    let eager = enabled || (!multipart && length.is_some_and(|n| n > 0 && n <= SMALL_BODY));
132    let mut request_body = Vec::new();
133    if eager {
134        let limit = request.extensions().get::<crate::body_limit::Limit>().and_then(|l| l.0);
135        let body = std::mem::replace(request.body_mut(), Body::empty());
136        let bytes = match axum::body::to_bytes(body, usize::MAX).await {
137            Ok(bytes) => bytes,
138            // Go answers before it logs anything.
139            Err(error) if crate::body_limit::is_too_large(&error) => return crate::body_limit::too_large(),
140            Err(error) => {
141                // Preserve the transport error downstream; never turn it into
142                // a successful empty body when capture cannot read the input.
143                *request.body_mut() = Body::from_stream(futures_util::stream::once(async move {
144                    Err::<Bytes, _>(io::Error::other(error.to_string()))
145                }));
146                return next.run(request).await;
147            }
148        };
149        *request.body_mut() = Body::from(bytes.clone());
150        request_body = decode(bytes.to_vec(), request.headers(), true, limit).await.0;
151    }
152    let deferred = if !eager && !multipart && length != Some(0) && !request.body().is_end_stream() {
153        Spool::new(&state.dir, "request-body").ok().map(|spool| {
154            Arc::new(Mutex::new(Deferred {
155                spool,
156                length,
157                read: 0,
158                captured: 0,
159                eof: false,
160                truncated: false,
161                failed: false,
162            }))
163        })
164    } else {
165        None
166    };
167    if let Some(deferred) = &deferred {
168        let body = std::mem::replace(request.body_mut(), Body::empty());
169        *request.body_mut() = Body::new(RequestBody {
170            body,
171            deferred: deferred.clone(),
172        });
173    }
174    let query = mask_query(request.uri().query().unwrap_or_default());
175    let url = if query.is_empty() {
176        path
177    } else {
178        format!("{path}?{query}")
179    };
180    let id = request
181        .extensions()
182        .get::<RequestId>()
183        .map(|id| id.0.clone())
184        .unwrap_or_default();
185    let sink = RequestLog(
186        Arc::new(Mutex::new(Some(Record {
187            state,
188            enabled,
189            url,
190            method: request.method().to_string(),
191            id,
192            timestamp: Local::now(),
193            headers: request.headers().clone(),
194            request_body,
195            deferred,
196            status: 200,
197            response_headers: HeaderMap::new(),
198            response: Vec::new(),
199            response_spool: None,
200            streaming: false,
201            streaming_failed: false,
202            detached: false,
203            metadata_ready: false,
204            websocket_done: false,
205            timeline: None,
206            api_timeline: Section::new("api-websocket-timeline"),
207            api_request: Section::new("api-request"),
208            deferred_api_request: Vec::new(),
209            api_response: Section::new("api-response"),
210            api_timestamp: None,
211            api_errors: Vec::new(),
212            upstream: UpstreamCapture::default(),
213            redact_sdp: false,
214        }))),
215        Arc::new(Finished::default()),
216    );
217    request.extensions_mut().insert(sink.clone());
218    let mut guard = Completion(Some(sink.clone()));
219    let head = request.method() == "HEAD";
220    let response = CURRENT.scope(sink.clone(), next.run(request)).await;
221    guard.0 = None;
222    capture_response(sink, head, response).await
223}
224
225async fn capture_response(sink: RequestLog, head: bool, mut response: Response) -> Response {
226    if response.extensions().get::<Captured>().is_some() {
227        return response;
228    }
229    response.extensions_mut().insert(Captured);
230    let mut guard = Completion(Some(sink.clone()));
231    let (rt, enabled) = {
232        let locked = sink.0.lock().unwrap_or_else(PoisonError::into_inner);
233        let Some(record) = locked.as_ref() else { return response };
234        (record.state.rt.clone(), record.enabled)
235    };
236    let current_enabled = setting(&rt.config(), "request-log").and_then(serde_yaml_ng::Value::as_bool) == Some(true);
237    if head && (current_enabled || (!enabled && response.status().as_u16() >= 400 && response.status().as_u16() != 499))
238    {
239        // Axum strips HEAD bodies after route middleware returns, so a lazy
240        // response-body tee would never see these handler writes. Gin logs them
241        // before net/http suppresses transport output; collect them here.
242        if let Some(record) = sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
243            record.status = response.status().as_u16();
244            record.response_headers = response.headers().clone();
245        }
246        let body = std::mem::replace(response.body_mut(), Body::empty());
247        if let Ok(bytes) = axum::body::to_bytes(body, usize::MAX).await {
248            if let Some(record) = sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
249                record.response.extend_from_slice(&bytes);
250            }
251            *response.body_mut() = Body::from(bytes);
252        }
253    }
254    let mut locked = sink.0.lock().unwrap_or_else(PoisonError::into_inner);
255    let Some(record) = locked.as_mut() else {
256        return response;
257    };
258    record.status = response.status().as_u16();
259    record.response_headers = response.headers().clone();
260    record.metadata_ready = true;
261    let content_type = response
262        .headers()
263        .get("content-type")
264        .and_then(|v| v.to_str().ok())
265        .unwrap_or_default();
266    record.streaming = current_enabled
267        && (content_type.contains("text/event-stream")
268            || (content_type.trim().is_empty()
269                && [b"\"stream\":true".as_slice(), b"\"stream\": true".as_slice()]
270                    .iter()
271                    .any(|needle| {
272                        record
273                            .request_body
274                            .windows(needle.len())
275                            .any(|window| window == *needle)
276                    })));
277    let chunks = if record.streaming && !head {
278        match Spool::new(&record.state.dir, "response-body") {
279            Ok(spool) => match spool.file.try_clone() {
280                Ok(file) => {
281                    record.response_spool = Some(spool);
282                    let (tx, rx) = tokio::sync::mpsc::channel::<Bytes>(100);
283                    tokio::spawn(spool_response(file, rx, sink.clone()));
284                    guard.0 = None;
285                    Some(tx)
286                }
287                Err(_) => None,
288            },
289            Err(_) => None,
290        }
291    } else {
292        None
293    };
294    // Only the error-only policy is fixed at entry; Go checks current logger
295    // enablement on every write, including reloads during a lazy response body.
296    let buffer = !enabled && record.status >= 400 && record.status != 499;
297    let complete_now = record.detached && record.websocket_done;
298    drop(locked);
299    if complete_now {
300        schedule(sink.clone(), false);
301    }
302    let (parts, body) = response.into_parts();
303    Response::from_parts(
304        parts,
305        Body::new(ResponseBody {
306            body,
307            sink,
308            guard,
309            chunks,
310            buffer,
311        }),
312    )
313}
314
315/// Cloneable per-request sink carried in HTTP extensions. Capture helpers must
316/// mask upstream credentials before passing preformatted API sections, as in Go.
317#[derive(Clone)]
318pub struct RequestLog(Arc<Mutex<Option<Record>>>, Arc<Finished>);
319
320#[derive(Default)]
321struct Finished {
322    done: AtomicBool,
323    notify: tokio::sync::Notify,
324}
325
326impl RequestLog {
327    /// Executors receive this through ExecRequest::capture(); WebSocket owners
328    /// can retain it across connection tasks, independently of task-local state.
329    pub fn capture_sink(&self) -> cpa_core::exec::CaptureSink {
330        cpa_core::exec::CaptureSink::new(Arc::new(self.clone()))
331    }
332
333    /// Full middleware UUID (the filename uses its trailing eight characters).
334    pub fn request_id(&self) -> String {
335        self.0
336            .lock()
337            .unwrap_or_else(PoisonError::into_inner)
338            .as_ref()
339            .map(|r| r.id.clone())
340            .unwrap_or_default()
341    }
342
343    /// Transfer finalization to the connection task before returning HTTP 101.
344    /// Returns None for disabled logging, a second detach, or an already finalized
345    /// request. This keeps disabled websocket logging free of disk/body capture.
346    pub fn detach_websocket(&self) -> Option<WebsocketLog> {
347        let mut locked = self.0.lock().unwrap_or_else(PoisonError::into_inner);
348        let record = locked.as_mut()?;
349        if !record.enabled || record.detached {
350            return None;
351        }
352        record.detached = true;
353        Some(WebsocketLog(Some(self.clone())))
354    }
355
356    /// Logs this request's bodies, downstream and upstream, without SDP ICE credentials
357    /// (`cpa_exec::codex_live::redact_sdp`). The bytes on the wire are unchanged.
358    pub fn redact_sdp(&self) {
359        if let Some(record) = self.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
360            record.redact_sdp = true;
361        }
362    }
363
364    pub fn api_request(&self, bytes: &[u8]) {
365        if let Some(record) = self.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
366            record.api_request.append(&record.state.dir, bytes);
367        }
368    }
369
370    pub fn api_response(&self, bytes: &[u8], timestamp: DateTime<Local>) {
371        if let Some(record) = self.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
372            record.api_response.append(&record.state.dir, bytes);
373            record.api_timestamp.get_or_insert(timestamp);
374        }
375    }
376
377    pub fn api_websocket_timeline(&self, bytes: &[u8]) {
378        if let Some(record) = self.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
379            record.api_timeline.append(&record.state.dir, bytes);
380        }
381    }
382
383    /// Go API_RESPONSE_ERROR: actionable upstream failures can force an error
384    /// log even when the downstream HTTP response started with 200.
385    pub fn api_error(&self, status: u16, message: String) {
386        if let Some(record) = self.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
387            record.api_errors.push((status, message));
388        }
389    }
390
391    async fn finish(&self, websocket: bool) {
392        let record = {
393            let mut record = self.0.lock().unwrap_or_else(PoisonError::into_inner);
394            if let Some(record) = record.as_mut() {
395                record.websocket_done |= websocket;
396            }
397            if record
398                .as_ref()
399                .is_some_and(|r| r.detached && !(r.metadata_ready && r.websocket_done))
400            {
401                None
402            } else {
403                record.take()
404            }
405        };
406        if let Some(record) = record {
407            record.finish().await;
408            self.1.done.store(true, Ordering::Release);
409            self.1.notify.notify_waiters();
410        }
411        if websocket {
412            self.finished().await;
413        }
414    }
415
416    /// Resolves once this request's capture is finalized: its log written, or its
417    /// captured bytes dropped when no log is due. For HTTP that happens in a task of its
418    /// own after the response's last byte (or its drop).
419    pub async fn finished(&self) {
420        loop {
421            let notified = self.1.notify.notified();
422            tokio::pin!(notified);
423            notified.as_mut().enable();
424            if self.1.done.load(Ordering::Acquire) {
425                break;
426            }
427            notified.await;
428        }
429    }
430}
431
432#[derive(Default)]
433struct UpstreamCapture {
434    attempts: usize,
435    deferred_requests: usize,
436    deferred_bytes: usize,
437    intro: bool,
438    status: bool,
439    headers: bool,
440    body: bool,
441    event: bool,
442    error: bool,
443}
444
445fn commercial_mode(config: &cpa_core::config::Config) -> bool {
446    config
447        .document
448        .get("server")
449        .and_then(|s| s.get("commercial-mode"))
450        .and_then(serde_yaml_ng::Value::as_bool)
451        == Some(true)
452}
453
454fn request_log_on(config: &cpa_core::config::Config) -> bool {
455    setting(config, "request-log").and_then(serde_yaml_ng::Value::as_bool) == Some(true)
456}
457
458impl cpa_core::exec::CaptureObserver for RequestLog {
459    fn logs_responses(&self) -> bool {
460        let locked = self.0.lock().unwrap_or_else(PoisonError::into_inner);
461        locked.as_ref().is_some_and(|record| {
462            let config = record.state.rt.config();
463            !commercial_mode(&config) && request_log_on(&config)
464        })
465    }
466
467    fn record(&self, event: cpa_core::exec::CaptureEvent<'_>) {
468        self.capture(event, None);
469    }
470
471    fn record_request(&self, info: cpa_core::exec::UpstreamRequest<'_>, body: &Bytes) {
472        let info = cpa_core::exec::UpstreamRequest {
473            body: &body[..],
474            ..info
475        };
476        self.capture(cpa_core::exec::CaptureEvent::Request(info), Some(body));
477    }
478}
479
480impl RequestLog {
481    /// `shared` is the request event's body as the caller holds it: the deferred copy
482    /// for an error log keeps a handle to it instead of a copy.
483    fn capture(&self, event: cpa_core::exec::CaptureEvent<'_>, shared: Option<&Bytes>) {
484        use cpa_core::exec::CaptureEvent::*;
485        let mut locked = self.0.lock().unwrap_or_else(PoisonError::into_inner);
486        let Some(record) = locked.as_mut() else { return };
487        let config = record.state.rt.config();
488        if commercial_mode(&config) {
489            return;
490        }
491        let enabled = request_log_on(&config);
492        // Upstream bodies are redacted as they enter the log. The live call handler
493        // records each body whole, as one event (bounded by its read limits), so an SDP
494        // line never spans two events.
495        let redacted;
496        let shared = shared.filter(|_| !record.redact_sdp);
497        let event = match event {
498            Request(info) if record.redact_sdp => {
499                redacted = cpa_exec::codex_live::redact_sdp(info.body);
500                Request(cpa_core::exec::UpstreamRequest {
501                    body: &redacted,
502                    ..info
503                })
504            }
505            ResponseChunk(chunk) if record.redact_sdp => {
506                redacted = cpa_exec::codex_live::redact_sdp(chunk);
507                ResponseChunk(&redacted)
508            }
509            event => event,
510        };
511        match event {
512            Request(info) => {
513                let capture = &mut record.upstream;
514                let (index, limit) = if enabled {
515                    let index = capture.attempts + 1;
516                    let deferred_requests = capture.deferred_requests;
517                    let deferred_bytes = capture.deferred_bytes;
518                    *capture = UpstreamCapture {
519                        attempts: index,
520                        deferred_requests,
521                        deferred_bytes,
522                        ..Default::default()
523                    };
524                    (index, info.body.len())
525                } else {
526                    capture.deferred_requests += 1;
527                    let length = info
528                        .body
529                        .len()
530                        .min((DEFERRED_BODY as usize).saturating_sub(capture.deferred_bytes));
531                    capture.deferred_bytes += length;
532                    (capture.deferred_requests, length)
533                };
534                let mut head = Vec::new();
535                write!(
536                    head,
537                    "=== API REQUEST {index} ===\nTimestamp: {}\nUpstream URL: {}\n",
538                    timestamp(Local::now()),
539                    if info.url.is_empty() {
540                        String::from("<unknown>")
541                    } else {
542                        logged_url(info.url)
543                    }
544                )
545                .unwrap();
546                if !info.method.is_empty() {
547                    writeln!(head, "HTTP Method: {}", info.method).unwrap();
548                }
549                upstream_auth(&mut head, &info);
550                head.extend_from_slice(b"\nHeaders:\n");
551                upstream_headers(&mut head, info.headers);
552                head.extend_from_slice(b"\nBody:\n");
553                let body: &[u8] = if info.body.is_empty() {
554                    b"<empty>"
555                } else {
556                    &info.body[..limit]
557                };
558                let mut tail = Vec::new();
559                if limit < info.body.len() {
560                    write!(tail, "\n[API REQUEST BODY TRUNCATED: captured first {limit} bytes]").unwrap();
561                }
562                tail.extend_from_slice(b"\n\n");
563                if enabled {
564                    // The body goes straight to the section, so a large one is not
565                    // copied into a second buffer first.
566                    for part in [&head[..], body, &tail[..]] {
567                        record.api_request.append(&record.state.dir, part);
568                    }
569                } else {
570                    // Kept until the exchange ends, possibly for a whole stream: a
571                    // handle to the body being sent where the caller gave one and the
572                    // log keeps all of it. A slice would keep the whole body alive past
573                    // the cap, so a truncated body is copied.
574                    let body = match shared {
575                        Some(shared) if !shared.is_empty() && limit == shared.len() => shared.clone(),
576                        _ => Bytes::copy_from_slice(body),
577                    };
578                    record
579                        .deferred_api_request
580                        .extend([Bytes::from(head), body, Bytes::from(tail)]);
581                }
582            }
583            _ if !enabled => {}
584            ResponseMetadata(status, headers) => {
585                upstream_intro(record);
586                let mut part = Vec::new();
587                if status > 0 && !record.upstream.status {
588                    writeln!(part, "Status: {status}").unwrap();
589                    record.upstream.status = true;
590                }
591                if !record.upstream.headers {
592                    part.extend_from_slice(b"Headers:\n");
593                    upstream_headers(&mut part, headers);
594                    part.push(b'\n');
595                    record.upstream.headers = true;
596                }
597                record.api_response.append(&record.state.dir, &part);
598            }
599            ResponseError(error) => {
600                upstream_intro(record);
601                let mut part = Vec::new();
602                if record.upstream.error {
603                    part.push(b'\n');
604                }
605                writeln!(part, "Error: {}", logged_text(error)).unwrap();
606                record.api_response.append(&record.state.dir, &part);
607                record.upstream.error = true;
608            }
609            ResponseChunk(chunk) => {
610                let chunk = trim(chunk);
611                if chunk.is_empty() {
612                    return;
613                }
614                upstream_intro(record);
615                let dir = &record.state.dir;
616                if !record.upstream.headers {
617                    record.api_response.append(dir, b"Headers:\n<none>\n\n");
618                    record.upstream.headers = true;
619                }
620                if !record.upstream.body {
621                    record.api_response.append(dir, b"Body:\n");
622                } else {
623                    record.api_response.append(
624                        dir,
625                        if record.upstream.event && chunk.starts_with(b"data:") {
626                            b"\n"
627                        } else {
628                            b"\n\n"
629                        },
630                    );
631                }
632                record.api_response.append(dir, chunk);
633                record.upstream.body = true;
634                record.upstream.event = chunk.starts_with(b"event:");
635            }
636            WebsocketRequest(info) => {
637                let mut part = websocket_event("api.websocket.request");
638                if !info.url.is_empty() {
639                    writeln!(part, "Upstream URL: {}", logged_url(info.url)).unwrap();
640                }
641                upstream_auth(&mut part, &info);
642                part.extend_from_slice(b"Headers:\n");
643                upstream_headers(&mut part, info.headers);
644                part.extend_from_slice(b"\nBody:\n");
645                part.extend_from_slice(if info.body.is_empty() { b"<empty>" } else { info.body });
646                upstream_timeline(&mut record.api_timeline, &record.state.dir, &part);
647            }
648            WebsocketHandshake(status, headers) => {
649                let mut part = websocket_event("api.websocket.handshake");
650                if status > 0 {
651                    writeln!(part, "Status: {status}").unwrap();
652                }
653                part.extend_from_slice(b"Headers:\n");
654                upstream_headers(&mut part, headers);
655                upstream_timeline(&mut record.api_timeline, &record.state.dir, &part);
656            }
657            WebsocketResponse(payload) => {
658                let payload = trim(payload);
659                if payload.is_empty() {
660                    return;
661                }
662                record.api_timestamp.get_or_insert_with(Local::now);
663                let mut part = websocket_event("api.websocket.response");
664                part.extend_from_slice(payload);
665                upstream_timeline(&mut record.api_timeline, &record.state.dir, &part);
666            }
667            WebsocketError { stage, error } => {
668                record.api_timestamp.get_or_insert_with(Local::now);
669                let mut part = websocket_event("api.websocket.error");
670                let stage = crate::gojson::trim(stage);
671                if !stage.is_empty() {
672                    writeln!(part, "Stage: {stage}").unwrap();
673                }
674                writeln!(part, "Error: {}", logged_text(error)).unwrap();
675                upstream_timeline(&mut record.api_timeline, &record.state.dir, &part);
676            }
677        }
678    }
679}
680
681fn upstream_intro(record: &mut Record) {
682    if record.upstream.intro {
683        return;
684    }
685    if record.upstream.attempts == 0 {
686        record.upstream.attempts = 1;
687        record
688            .api_request
689            .append(&record.state.dir, b"=== API REQUEST 1 ===\n<missing>\n\n");
690    }
691    let mut part = Vec::new();
692    if !record.api_response.is_empty() {
693        part.resize(2usize.saturating_sub(record.api_response.trailing), b'\n');
694    }
695    write!(
696        part,
697        "=== API RESPONSE {} ===\nTimestamp: {}\n\n",
698        record.upstream.attempts,
699        timestamp(Local::now())
700    )
701    .unwrap();
702    record.api_response.append(&record.state.dir, &part);
703    record.upstream.intro = true;
704}
705
706fn upstream_headers(out: &mut Vec<u8>, headers: &[(String, String)]) {
707    if headers.is_empty() {
708        out.extend_from_slice(b"<none>\n");
709        return;
710    }
711    let mut headers: Vec<_> = headers.iter().collect();
712    headers.sort_by(|a, b| a.0.cmp(&b.0));
713    for (name, value) in headers {
714        write!(out, "{name}: ").unwrap();
715        out.extend_from_slice(&masked_header(&name.to_ascii_lowercase(), value.as_bytes()));
716        out.push(b'\n');
717    }
718}
719
720fn upstream_auth(out: &mut Vec<u8>, info: &cpa_core::exec::UpstreamRequest<'_>) {
721    let mut parts = Vec::new();
722    for (label, value) in [
723        ("provider", info.provider),
724        ("auth_id", info.auth_id),
725        ("label", info.auth_label),
726    ] {
727        let value = crate::gojson::trim(value);
728        if !value.is_empty() {
729            parts.push(format!("{label}={value}"));
730        }
731    }
732    let kind = cpa_common::gostr::lower_bytes(crate::gojson::trim(info.auth_type).as_bytes());
733    let value = crate::gojson::trim(info.auth_value);
734    if !kind.is_empty() {
735        let mut text = format!("type={kind}");
736        if kind != "oauth" && !value.is_empty() {
737            let value = if kind == "api_key" {
738                String::from_utf8_lossy(&hide_key(value.as_bytes())).into_owned()
739            } else {
740                value.into()
741            };
742            text.push_str(&format!(" value={value}"));
743        }
744        parts.push(text);
745    }
746    if !parts.is_empty() {
747        writeln!(out, "Auth: {}", parts.join(", ")).unwrap();
748    }
749}
750
751fn websocket_event(event: &str) -> Vec<u8> {
752    format!("Timestamp: {}\nEvent: {event}\n", timestamp(Local::now())).into_bytes()
753}
754
755fn upstream_timeline(out: &mut Section, dir: &Path, part: &[u8]) {
756    if !out.is_empty() {
757        out.append(dir, b"\n\n");
758    }
759    out.append(dir, trim(part));
760}
761
762/// Connection-owned timeline sink. Explicit `close().await` waits for the durable
763/// log; dropping the sink also schedules finalization on client disconnect.
764pub struct WebsocketLog(Option<RequestLog>);
765
766impl WebsocketLog {
767    pub fn append_part(&self, part: &[u8]) -> io::Result<()> {
768        if trim(part).is_empty() {
769            return Ok(());
770        }
771        if let Some(sink) = &self.0 {
772            let mut record = sink.0.lock().unwrap_or_else(PoisonError::into_inner);
773            if let Some(record) = record.as_mut() {
774                if record.timeline.is_none() {
775                    record.timeline = Some(Spool::new(&record.state.dir, "websocket-timeline")?);
776                }
777                let spool = record.timeline.as_mut().unwrap();
778                if spool.file.metadata()?.len() > 0 {
779                    spool.file.write_all(b"\n")?;
780                }
781                spool.file.write_all(part)?;
782                if !part.ends_with(b"\n") {
783                    spool.file.write_all(b"\n")?;
784                }
785            } else {
786                return Err(io::Error::new(io::ErrorKind::BrokenPipe, "request log finalized"));
787            }
788        }
789        Ok(())
790    }
791
792    pub async fn close(mut self) {
793        if let Some(sink) = self.0.take() {
794            // Disconnect cancellation must not cancel the final delivery itself.
795            let _ = tokio::spawn(async move { sink.finish(true).await }).await;
796        }
797    }
798}
799
800impl Drop for WebsocketLog {
801    fn drop(&mut self) {
802        if let Some(sink) = self.0.take() {
803            schedule(sink, true);
804        }
805    }
806}
807
808fn schedule(sink: RequestLog, websocket: bool) {
809    if let Ok(handle) = tokio::runtime::Handle::try_current() {
810        handle.spawn(async move {
811            sink.finish(websocket).await;
812        });
813    }
814}
815
816/// Drain even after a sticky disk error. Only EOF/drop ends capture; never
817/// finalize a live HTTP stream merely because its log file could not be written.
818async fn spool_response(file: File, mut rx: tokio::sync::mpsc::Receiver<Bytes>, sink: RequestLog) {
819    let mut file = tokio::fs::File::from_std(file);
820    let mut failed = false;
821    while let Some(bytes) = rx.recv().await {
822        if !failed && file.write_all(&bytes).await.is_err() {
823            failed = true;
824        }
825    }
826    failed |= file.flush().await.is_err();
827    drop(file);
828    if let Some(record) = sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
829        record.streaming_failed = failed;
830    }
831    sink.finish(false).await;
832}
833
834struct Completion(Option<RequestLog>);
835impl Drop for Completion {
836    fn drop(&mut self) {
837        if let Some(sink) = self.0.take() {
838            if let Some(record) = sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
839                // A cancelled handler has Go's default unwritten 200; release
840                // any connection-close waiter even without response headers.
841                record.metadata_ready = true;
842            }
843            schedule(sink, false);
844        }
845    }
846}
847
848struct ResponseBody {
849    body: Body,
850    sink: RequestLog,
851    guard: Completion,
852    chunks: Option<tokio::sync::mpsc::Sender<Bytes>>,
853    buffer: bool,
854}
855
856impl HttpBody for ResponseBody {
857    type Data = Bytes;
858    type Error = axum::Error;
859    fn poll_frame(
860        mut self: Pin<&mut Self>,
861        cx: &mut Context<'_>,
862    ) -> Poll<Option<Result<hyper::body::Frame<Bytes>, axum::Error>>> {
863        let result = Pin::new(&mut self.body).poll_frame(cx);
864        if let Poll::Ready(Some(Ok(frame))) = &result
865            && let Some(bytes) = frame.data_ref()
866        {
867            if let Some(chunks) = &self.chunks {
868                let _ = chunks.try_send(bytes.clone());
869                if let Some(record) = self.sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
870                    record.api_timestamp.get_or_insert_with(Local::now);
871                }
872            } else if let Some(record) = self.sink.0.lock().unwrap_or_else(PoisonError::into_inner).as_mut() {
873                let config = record.state.rt.config();
874                if self.buffer || setting(&config, "request-log").and_then(serde_yaml_ng::Value::as_bool) == Some(true)
875                {
876                    record.response.extend_from_slice(bytes);
877                }
878            }
879        }
880        if matches!(result, Poll::Ready(None | Some(Err(_)))) {
881            self.chunks.take();
882            if let Some(sink) = self.guard.0.take() {
883                schedule(sink, false);
884            }
885        }
886        result
887    }
888    fn is_end_stream(&self) -> bool {
889        self.body.is_end_stream()
890    }
891    fn size_hint(&self) -> hyper::body::SizeHint {
892        self.body.size_hint()
893    }
894}
895
896struct RequestBody {
897    body: Body,
898    deferred: Arc<Mutex<Deferred>>,
899}
900impl HttpBody for RequestBody {
901    type Data = Bytes;
902    type Error = axum::Error;
903    fn poll_frame(
904        mut self: Pin<&mut Self>,
905        cx: &mut Context<'_>,
906    ) -> Poll<Option<Result<hyper::body::Frame<Bytes>, axum::Error>>> {
907        let result = Pin::new(&mut self.body).poll_frame(cx);
908        let mut deferred = self.deferred.lock().unwrap_or_else(PoisonError::into_inner);
909        if let Poll::Ready(Some(Ok(frame))) = &result
910            && let Some(bytes) = frame.data_ref()
911        {
912            deferred.read += bytes.len() as u64;
913            let length = bytes
914                .len()
915                .min(DEFERRED_BODY.saturating_sub(deferred.captured) as usize);
916            deferred.truncated |= length < bytes.len();
917            if !deferred.failed {
918                match deferred.spool.file.write(&bytes[..length]) {
919                    Ok(n) => {
920                        deferred.captured += n as u64;
921                        deferred.failed = n < length;
922                    }
923                    Err(_) => deferred.failed = true,
924                }
925            }
926        }
927        if matches!(result, Poll::Ready(None)) {
928            deferred.eof = true;
929        }
930        result
931    }
932    fn is_end_stream(&self) -> bool {
933        self.body.is_end_stream()
934    }
935    fn size_hint(&self) -> hyper::body::SizeHint {
936        self.body.size_hint()
937    }
938}
939
940struct Deferred {
941    spool: Spool,
942    length: Option<u64>,
943    read: u64,
944    captured: u64,
945    eof: bool,
946    truncated: bool,
947    failed: bool,
948}
949impl Deferred {
950    fn bytes(&self) -> io::Result<(Vec<u8>, String)> {
951        if self.failed {
952            return Err(io::Error::other("request body capture failed"));
953        }
954        let mut markers = Vec::new();
955        if self.truncated {
956            markers.push(format!(
957                "[REQUEST BODY TRUNCATED: captured first {} bytes]",
958                self.captured
959            ));
960        }
961        if !(self.eof || self.length.is_some_and(|n| self.read >= n)) {
962            markers.push(match self.length {
963                Some(n) => format!("[REQUEST BODY CAPTURE INCOMPLETE: consumed {} of {n} bytes]", self.read),
964                None => format!(
965                    "[REQUEST BODY CAPTURE INCOMPLETE: consumed {} bytes from an unknown-length body]",
966                    self.read
967                ),
968            });
969        }
970        Ok((std::fs::read(&self.spool.path)?, markers.join("\n")))
971    }
972}
973
974struct Spool {
975    file: File,
976    path: PathBuf,
977}
978impl Spool {
979    fn new(dir: &Path, prefix: &str) -> io::Result<Self> {
980        std::fs::create_dir_all(dir)?;
981        let path = dir.join(format!("{prefix}-{}.tmp", uuid::Uuid::new_v4()));
982        let mut options = OpenOptions::new();
983        options.write(true).create_new(true);
984        // Go's os.OpenFile modes; Windows ignores them.
985        #[cfg(unix)]
986        std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
987        let file = options.open(&path)?;
988        Ok(Self { file, path })
989    }
990}
991impl Drop for Spool {
992    fn drop(&mut self) {
993        let _ = std::fs::remove_file(&self.path);
994    }
995}
996
997/// In-memory bytes an upstream section keeps before it moves to its spool file.
998const SECTION_MEMORY: usize = 64 << 10;
999/// Long enough for every section header that `section` looks for.
1000const SECTION_HEAD: usize = 32;
1001
1002/// An append-only upstream section of the log (API REQUEST, API RESPONSE or
1003/// API WEBSOCKET TIMELINE). Go appends these to file-backed sources
1004/// (request_logging.go attachRequestLogSources), so a long stream or WebSocket
1005/// session does not grow in memory. Here a section stays in memory until it
1006/// passes SECTION_MEMORY bytes, then the buffered bytes and everything after
1007/// them go to a spool file in the log directory. If the spool cannot be
1008/// written, the section keeps the rest in memory, as Go's writeAttemptResponse
1009/// falls back to a strings.Builder.
1010struct Section {
1011    prefix: &'static str,
1012    memory: Vec<u8>,
1013    spool: Option<Spool>,
1014    /// Bytes at the start of the spool file that belong to the section; a
1015    /// failed write can leave more after them, which are never read.
1016    spilled: u64,
1017    spill_failed: bool,
1018    /// The first SECTION_HEAD bytes of the section.
1019    head: Vec<u8>,
1020    /// Newlines at the end of the section.
1021    trailing: usize,
1022    /// Every byte so far is whitespace (the section trims to nothing).
1023    blank: bool,
1024}
1025
1026impl Section {
1027    fn new(prefix: &'static str) -> Self {
1028        Self {
1029            prefix,
1030            memory: Vec::new(),
1031            spool: None,
1032            spilled: 0,
1033            spill_failed: false,
1034            head: Vec::new(),
1035            trailing: 0,
1036            blank: true,
1037        }
1038    }
1039
1040    /// A section that holds `bytes` in memory, for a buffer that is already
1041    /// in memory as a whole.
1042    fn from_vec(prefix: &'static str, bytes: Vec<u8>) -> Self {
1043        let mut section = Self::new(prefix);
1044        section.track(&bytes);
1045        section.memory = bytes;
1046        section
1047    }
1048
1049    fn is_empty(&self) -> bool {
1050        self.spilled == 0 && self.memory.is_empty()
1051    }
1052
1053    fn track(&mut self, bytes: &[u8]) {
1054        if bytes.is_empty() {
1055            return;
1056        }
1057        let room = SECTION_HEAD.saturating_sub(self.head.len());
1058        self.head.extend_from_slice(&bytes[..room.min(bytes.len())]);
1059        let newlines = bytes.iter().rev().take_while(|&&b| b == b'\n').count();
1060        self.trailing = if newlines == bytes.len() {
1061            self.trailing + newlines
1062        } else {
1063            newlines
1064        };
1065        self.blank &= trim(bytes).is_empty();
1066    }
1067
1068    fn append(&mut self, dir: &Path, bytes: &[u8]) {
1069        if bytes.is_empty() {
1070            return;
1071        }
1072        self.track(bytes);
1073        if !self.spill_failed && self.memory.len() + bytes.len() > SECTION_MEMORY {
1074            if self.spill(dir, bytes).is_ok() {
1075                return;
1076            }
1077            self.spill_failed = true;
1078        }
1079        self.memory.extend_from_slice(bytes);
1080    }
1081
1082    /// Moves the buffered bytes and then `bytes` to the spool file. On error,
1083    /// whatever did not reach the file is still in memory or in `bytes`.
1084    fn spill(&mut self, dir: &Path, bytes: &[u8]) -> io::Result<()> {
1085        if self.spool.is_none() {
1086            self.spool = Some(Spool::new(dir, self.prefix)?);
1087        }
1088        let file = &mut self.spool.as_mut().unwrap().file;
1089        file.write_all(&self.memory)?;
1090        self.spilled += self.memory.len() as u64;
1091        // Appends only reach memory while it stays within SECTION_MEMORY, so
1092        // the kept allocation is bounded too.
1093        self.memory.clear();
1094        file.write_all(bytes)?;
1095        self.spilled += bytes.len() as u64;
1096        Ok(())
1097    }
1098
1099    fn write_to(&self, out: &mut dyn Write) -> io::Result<()> {
1100        if let Some(spool) = self.spool.as_ref().filter(|_| self.spilled > 0) {
1101            let copied = io::copy(&mut File::open(&spool.path)?.take(self.spilled), out)?;
1102            if copied != self.spilled {
1103                return Err(io::Error::new(
1104                    io::ErrorKind::UnexpectedEof,
1105                    format!("{} spool is shorter than written", self.prefix),
1106                ));
1107            }
1108        }
1109        out.write_all(&self.memory)
1110    }
1111}
1112
1113struct Record {
1114    state: CaptureState,
1115    enabled: bool,
1116    url: String,
1117    method: String,
1118    id: String,
1119    timestamp: DateTime<Local>,
1120    headers: HeaderMap,
1121    request_body: Vec<u8>,
1122    deferred: Option<Arc<Mutex<Deferred>>>,
1123    status: u16,
1124    response_headers: HeaderMap,
1125    response: Vec<u8>,
1126    response_spool: Option<Spool>,
1127    streaming: bool,
1128    streaming_failed: bool,
1129    detached: bool,
1130    metadata_ready: bool,
1131    websocket_done: bool,
1132    timeline: Option<Spool>,
1133    api_timeline: Section,
1134    api_request: Section,
1135    deferred_api_request: Vec<Bytes>,
1136    api_response: Section,
1137    api_timestamp: Option<DateTime<Local>>,
1138    api_errors: Vec<(u16, String)>,
1139    upstream: UpstreamCapture,
1140    /// Set by the live call handler: bodies are logged without SDP ICE credentials,
1141    /// upstream ones as they are recorded and downstream ones when the log is written.
1142    redact_sdp: bool,
1143}
1144
1145impl Record {
1146    async fn finish(mut self) {
1147        if self.streaming_failed {
1148            return;
1149        }
1150        let config = self.state.rt.config();
1151        let enabled = setting(&config, "request-log").and_then(serde_yaml_ng::Value::as_bool) == Some(true);
1152        let error = self.api_errors.iter().any(|(status, message)| {
1153            let lower = message.go_lower();
1154            *status != 499 && !lower.contains("context canceled") && !lower.contains("client closed request")
1155        }) || (self.status >= 400 && self.status != 499);
1156        let forced = !self.enabled && error && !enabled;
1157        if !enabled && !forced {
1158            return;
1159        }
1160        if forced && self.api_request.is_empty() {
1161            self.api_request =
1162                Section::from_vec("api-request", std::mem::take(&mut self.deferred_api_request).concat());
1163        }
1164        if let Some(deferred) = &self.deferred {
1165            let body = deferred.lock().unwrap_or_else(PoisonError::into_inner).bytes();
1166            if let Ok((body, marker)) = body {
1167                self.request_body = decode(body, &self.headers, true, Some(DEFERRED_BODY)).await.0;
1168                if !marker.is_empty() {
1169                    if !self.request_body.is_empty() && !self.request_body.ends_with(b"\n") {
1170                        self.request_body.push(b'\n');
1171                    }
1172                    self.request_body.extend_from_slice(marker.as_bytes());
1173                }
1174            }
1175        }
1176        let decompress_error = if !self.streaming {
1177            let (body, error) = decode(std::mem::take(&mut self.response), &self.response_headers, false, None).await;
1178            self.response = body;
1179            error
1180        } else {
1181            None
1182        };
1183        if self.redact_sdp {
1184            for body in [&mut self.request_body, &mut self.response] {
1185                if let std::borrow::Cow::Owned(redacted) = cpa_exec::codex_live::redact_sdp(body) {
1186                    *body = redacted;
1187                }
1188            }
1189        }
1190        if enabled && let Some(home) = self.state.rt.remote_dispatch() {
1191            if home.available() {
1192                let mut content = Vec::new();
1193                if self.format(&mut content, decompress_error).is_ok() {
1194                    let mut payload = serde_json::Map::new();
1195                    let mut headers: std::collections::BTreeMap<String, Vec<String>> = Default::default();
1196                    for (name, value) in &self.headers {
1197                        headers
1198                            .entry(header_name(name.as_str()))
1199                            .or_default()
1200                            .push(String::from_utf8_lossy(value.as_bytes()).into_owned());
1201                    }
1202                    if !headers.is_empty() {
1203                        payload.insert("headers".into(), serde_json::to_value(headers).unwrap());
1204                    }
1205                    if !self.id.trim().is_empty() {
1206                        payload.insert("request_id".into(), self.id.trim().into());
1207                    }
1208                    if !content.is_empty() {
1209                        payload.insert(
1210                            "request_log".into(),
1211                            String::from_utf8_lossy(&content).into_owned().into(),
1212                        );
1213                    }
1214                    if let Err(error) = home
1215                        .request_log(crate::gojson::sorted(&payload.into()).into_bytes())
1216                        .await
1217                    {
1218                        tracing::warn!("failed to forward request log to home: {error}");
1219                    }
1220                }
1221            }
1222            return;
1223        }
1224        let max = setting(&config, "error-logs-max-files")
1225            .and_then(serde_yaml_ng::Value::as_i64)
1226            .unwrap_or(10);
1227        let max = if max < 0 { 10 } else { max };
1228        let _ = tokio::task::spawn_blocking(move || {
1229            if let Err(error) = self.write(forced, max, decompress_error) {
1230                tracing::warn!("failed to write request log: {error}");
1231            }
1232        })
1233        .await;
1234    }
1235
1236    fn write(self, forced: bool, max: i64, decompress_error: Option<String>) -> io::Result<()> {
1237        std::fs::create_dir_all(&self.state.dir)?;
1238        let filename = filename(&self.url, &self.id, Local::now(), forced);
1239        let mut file = unique_file(&self.state.dir, &filename)?;
1240        self.format(&mut file, decompress_error)?;
1241        drop(file);
1242        if forced {
1243            retain_errors(&self.state.dir, max)?;
1244        }
1245        Ok(())
1246    }
1247
1248    fn format(&self, out: &mut dyn Write, decompress_error: Option<String>) -> io::Result<()> {
1249        let websocket = self
1250            .timeline
1251            .as_ref()
1252            .is_some_and(|s| s.file.metadata().is_ok_and(|m| m.len() > 0));
1253        let downstream = if websocket
1254            || self
1255                .headers
1256                .get("upgrade")
1257                .is_some_and(|v| trim(v.as_bytes()).eq_ignore_ascii_case(b"websocket"))
1258        {
1259            "websocket"
1260        } else {
1261            "http"
1262        };
1263        let http = !self.api_request.blank || !self.api_response.blank;
1264        let ws = !self.api_timeline.blank;
1265        let upstream = match (http, ws) {
1266            (true, true) => "websocket+http",
1267            (true, false) => "http",
1268            (false, true) => "websocket",
1269            _ => "",
1270        };
1271        write!(
1272            out,
1273            "=== REQUEST INFO ===\nVersion: {}\nURL: {}\nMethod: {}\nDownstream Transport: {downstream}\n",
1274            concat!("cliproxy-rs-", env!("CARGO_PKG_VERSION")),
1275            self.url,
1276            self.method
1277        )?;
1278        if !upstream.is_empty() {
1279            writeln!(out, "Upstream Transport: {upstream}")?;
1280        }
1281        write!(out, "Timestamp: {}\n\n\n=== HEADERS ===\n", timestamp(self.timestamp))?;
1282        headers(out, &self.headers, true)?;
1283        out.write_all(b"\n\n")?;
1284        if !websocket {
1285            out.write_all(b"=== REQUEST BODY ===\n")?;
1286            out.write_all(&self.request_body)?;
1287            spacing(out, &self.request_body)?;
1288        }
1289        if let Some(spool) = &self.timeline {
1290            out.write_all(b"=== WEBSOCKET TIMELINE ===\n")?;
1291            let mut input = File::open(&spool.path)?;
1292            let mut buffer = [0; 8192];
1293            let mut trailing = 0;
1294            loop {
1295                let n = input.read(&mut buffer)?;
1296                if n == 0 {
1297                    break;
1298                }
1299                out.write_all(&buffer[..n])?;
1300                let count = buffer[..n].iter().rev().take_while(|&&b| b == b'\n').count();
1301                trailing = if count == n { trailing + count } else { count };
1302            }
1303            for _ in trailing..3 {
1304                out.write_all(b"\n")?;
1305            }
1306        }
1307        section(out, "API WEBSOCKET TIMELINE", &self.api_timeline, None)?;
1308        section(out, "API REQUEST", &self.api_request, None)?;
1309        if !self.streaming {
1310            for (status, error) in &self.api_errors {
1311                write!(out, "=== API ERROR RESPONSE ===\nHTTP Status: {status}\n{error}")?;
1312                spacing(out, error.as_bytes())?;
1313            }
1314        }
1315        section(out, "API RESPONSE", &self.api_response, self.api_timestamp)?;
1316        if websocket {
1317            return Ok(());
1318        }
1319        write!(out, "=== RESPONSE ===\nStatus: {}\n", self.status)?;
1320        headers(out, &self.response_headers, false)?;
1321        if let Some(spool) = &self.response_spool
1322            && self.redact_sdp
1323        {
1324            // A live call's response is at most `live::MAX_BODY`; redacting it whole
1325            // also covers a marker split across written chunks.
1326            let body = std::fs::read(&spool.path)?;
1327            let body = cpa_exec::codex_live::redact_sdp(&body);
1328            if !(body.starts_with(b"\n") || body.starts_with(b"\r\n")) {
1329                out.write_all(b"\n")?;
1330            }
1331            out.write_all(&body)?;
1332        } else if let Some(spool) = &self.response_spool {
1333            let mut input = File::open(&spool.path)?;
1334            let mut first = [0; 2];
1335            let n = input.read(&mut first)?;
1336            if !(first[..n].starts_with(b"\n") || first[..n].starts_with(b"\r\n")) {
1337                out.write_all(b"\n")?;
1338            }
1339            out.write_all(&first[..n])?;
1340            io::copy(&mut input, out)?;
1341        } else {
1342            if !(self.response.starts_with(b"\n") || self.response.starts_with(b"\r\n")) {
1343                out.write_all(b"\n")?;
1344            }
1345            out.write_all(&self.response)?;
1346            if let Some(error) = decompress_error {
1347                write!(out, "\n[DECOMPRESSION ERROR: {error}]")?;
1348            }
1349        }
1350        if !self.streaming {
1351            out.write_all(b"\n")?;
1352        }
1353        Ok(())
1354    }
1355}
1356
1357fn trim(bytes: &[u8]) -> &[u8] {
1358    cpa_core::config::go_trim_space(bytes)
1359}
1360pub(crate) fn timestamp(at: DateTime<Local>) -> String {
1361    let text = at.to_rfc3339_opts(SecondsFormat::Nanos, true);
1362    let zone = &text[29..];
1363    let fraction = text[20..29].trim_end_matches('0');
1364    if fraction.is_empty() {
1365        format!("{}{zone}", &text[..19])
1366    } else {
1367        format!("{}.{fraction}{zone}", &text[..19])
1368    }
1369}
1370
1371fn header_name(name: &str) -> String {
1372    name.split('-')
1373        .map(|part| {
1374            let mut bytes = part.as_bytes().to_owned();
1375            if let Some(first) = bytes.first_mut() {
1376                first.make_ascii_uppercase();
1377            }
1378            String::from_utf8(bytes).unwrap_or_default()
1379        })
1380        .collect::<Vec<_>>()
1381        .join("-")
1382}
1383
1384/// Written in place of a cookie header value or URL userinfo.
1385const REDACTED: &str = "<redacted>";
1386
1387/// Deliberate difference: Go logs Cookie and Set-Cookie values unchanged
1388/// (util.MaskSensitiveHeaderValue). A session cookie is a credential, so the
1389/// whole value of any header whose lowercase name contains "cookie" is
1390/// replaced, on both the masked and the unmasked header sections.
1391fn cookie(name: &str) -> bool {
1392    name.contains("cookie")
1393}
1394
1395/// Deliberate difference: Go prints upstream URLs unchanged, and a configured
1396/// base-url can carry credentials. The logged copy replaces any userinfo and
1397/// masks sensitive query values the way the downstream URL is masked; the
1398/// request itself still goes to the URL as configured.
1399fn logged_url(url: &str) -> String {
1400    let mut out = String::new();
1401    let mut rest = url;
1402    // The authority starts after `scheme://`, or after a scheme-relative `//`.
1403    let authority_start = if url.starts_with("//") {
1404        Some(2)
1405    } else {
1406        url.find("://")
1407            .filter(|&i| {
1408                i > 0
1409                    && url[..i]
1410                        .bytes()
1411                        .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'))
1412            })
1413            .map(|i| i + 3)
1414    };
1415    if let Some(start) = authority_start {
1416        out.push_str(&url[..start]);
1417        rest = &url[start..];
1418        // Look for '@' up to the first '/', even past a '?' or '#': a
1419        // password with those characters unescaped still counts as userinfo,
1420        // and dropping too much is safer than printing a secret.
1421        let authority = &rest[..rest.find('/').unwrap_or(rest.len())];
1422        if let Some(at) = authority.rfind('@') {
1423            out.push_str(REDACTED);
1424            out.push('@');
1425            rest = &rest[at + 1..];
1426        }
1427    }
1428    let (rest, fragment) = rest.split_at(rest.find('#').unwrap_or(rest.len()));
1429    match rest.split_once('?') {
1430        Some((base, query)) => {
1431            out.push_str(base);
1432            out.push('?');
1433            out.push_str(&mask_upstream_query(query));
1434        }
1435        None => out.push_str(rest),
1436    }
1437    out.push_str(fragment);
1438    out
1439}
1440
1441/// Error text with every URL in it passed through [`logged_url`]: transport errors
1442/// can quote the upstream URL, query key included. A URL runs to the next whitespace,
1443/// quote or angle bracket, less trailing sentence punctuation; punctuation inside it is
1444/// masked with it, and a span holding a second URL is replaced whole.
1445fn logged_text(text: &str) -> String {
1446    let scheme_char = |c: char| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.');
1447    // The start of the scheme ending at byte `i` of `s`.
1448    let scheme_start = |s: &str, i: usize| {
1449        s[..i]
1450            .char_indices()
1451            .rev()
1452            .find(|&(_, c)| !scheme_char(c))
1453            .map_or(0, |(j, c)| j + c.len_utf8())
1454    };
1455    let mut out = String::new();
1456    let mut rest = text;
1457    while let Some(i) = rest.find("://") {
1458        let start = scheme_start(rest, i);
1459        let after = i + 3;
1460        let mut end = rest[after..]
1461            .find(|c: char| c.is_whitespace() || matches!(c, '"' | '<' | '>' | '`'))
1462            .map_or(rest.len(), |j| after + j);
1463        // Sentence punctuation after the URL stays in the text.
1464        end = (start + rest[start..end].trim_end_matches([')', ',', '.', ';', ':', '\'']).len()).max(after);
1465        out.push_str(&rest[..start]);
1466        let masked = logged_url(&rest[start..end]);
1467        // Another URL glued to this one (`https://a/x,https://u:p@b/y?token=t`) was not
1468        // masked as a URL of its own, so the whole span goes.
1469        let head = masked.find("://").map_or(masked.len(), |j| j + 3);
1470        if masked[head..].contains("://") {
1471            out.push_str(REDACTED);
1472        } else {
1473            out.push_str(&masked);
1474        }
1475        rest = &rest[end..];
1476    }
1477    out.push_str(rest);
1478    out
1479}
1480
1481fn masked_header(name: &str, value: &[u8]) -> Vec<u8> {
1482    if cookie(name) {
1483        return REDACTED.as_bytes().to_vec();
1484    }
1485    if name.contains("authorization") {
1486        let trimmed = trim(value);
1487        if let Some(i) = trimmed.iter().position(|b| *b == b' ') {
1488            return [&trimmed[..i + 1], &hide_key(&trimmed[i + 1..])].concat();
1489        }
1490        return hide_key(value);
1491    }
1492    if ["api-key", "apikey", "token", "secret"]
1493        .iter()
1494        .any(|key| name.contains(key))
1495    {
1496        hide_key(value)
1497    } else {
1498        value.to_vec()
1499    }
1500}
1501
1502fn headers(out: &mut dyn Write, headers: &HeaderMap, mask: bool) -> io::Result<()> {
1503    for (name, value) in headers {
1504        write!(out, "{}: ", header_name(name.as_str()))?;
1505        if mask {
1506            out.write_all(&masked_header(name.as_str(), value.as_bytes()))?;
1507        } else if cookie(name.as_str()) {
1508            out.write_all(REDACTED.as_bytes())?;
1509        } else {
1510            out.write_all(value.as_bytes())?;
1511        }
1512        out.write_all(b"\n")?;
1513    }
1514    Ok(())
1515}
1516
1517fn spacing(out: &mut dyn Write, bytes: &[u8]) -> io::Result<()> {
1518    pad(
1519        out,
1520        if bytes.is_empty() {
1521            1
1522        } else {
1523            bytes.iter().rev().take_while(|&&b| b == b'\n').count()
1524        },
1525    )
1526}
1527
1528fn pad(out: &mut dyn Write, trailing: usize) -> io::Result<()> {
1529    for _ in trailing..3 {
1530        out.write_all(b"\n")?;
1531    }
1532    Ok(())
1533}
1534
1535fn section(out: &mut dyn Write, name: &str, bytes: &Section, at: Option<DateTime<Local>>) -> io::Result<()> {
1536    if bytes.is_empty() {
1537        return Ok(());
1538    }
1539    if !bytes.head.starts_with(format!("=== {name}").as_bytes()) {
1540        writeln!(out, "=== {name} ===")?;
1541        if let Some(at) = at {
1542            writeln!(out, "Timestamp: {}", timestamp(at))?;
1543        }
1544    }
1545    bytes.write_to(out)?;
1546    pad(out, bytes.trailing)
1547}
1548
1549fn filename(url: &str, id: &str, at: DateTime<Local>, error: bool) -> String {
1550    let path = url
1551        .split('?')
1552        .next()
1553        .unwrap_or_default()
1554        .strip_prefix('/')
1555        .unwrap_or(url.split('?').next().unwrap_or_default());
1556    let mut sanitized = String::new();
1557    for ch in path.chars() {
1558        let ch = if matches!(
1559            ch,
1560            '/' | ':' | '<' | '>' | '"' | '|' | '?' | '*' | ' ' | '\t' | '\n' | '\r' | '\x0c'
1561        ) {
1562            '-'
1563        } else {
1564            ch
1565        };
1566        if ch != '-' || !sanitized.ends_with('-') {
1567            sanitized.push(ch);
1568        }
1569    }
1570    let path = sanitized.trim_matches('-');
1571    let path = if path.is_empty() { "root" } else { path };
1572    let id = if id.is_empty() {
1573        NEXT_ID.fetch_add(1, Ordering::Relaxed).wrapping_add(1).to_string()
1574    } else {
1575        id[id.len().saturating_sub(8)..].to_owned()
1576    };
1577    format!(
1578        "{}{path}-{}-{id}.log",
1579        if error { "error-" } else { "" },
1580        at.format("%Y-%m-%dT%H%M%S")
1581    )
1582}
1583
1584fn unique_file(dir: &Path, filename: &str) -> io::Result<File> {
1585    let (prefix, id) = filename
1586        .trim_end_matches(".log")
1587        .rsplit_once('-')
1588        .unwrap_or((filename, ""));
1589    for index in 0..=1000 {
1590        let name = if index == 0 {
1591            filename.to_owned()
1592        } else {
1593            format!("{prefix}_{index}-{id}.log")
1594        };
1595        let mut options = OpenOptions::new();
1596        options.create_new(true).write(true);
1597        // Deliberate difference: Go writes request logs 0644. They hold prompts and
1598        // responses, so only the owner may read them, like the spool files above.
1599        #[cfg(unix)]
1600        std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
1601        match options.open(dir.join(name)) {
1602            Ok(file) => return Ok(file),
1603            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => (),
1604            Err(error) => return Err(error),
1605        }
1606    }
1607    Err(io::Error::other(format!(
1608        "too many conflicting log files for {filename}"
1609    )))
1610}
1611
1612fn retain_errors(dir: &Path, max: i64) -> io::Result<()> {
1613    if max <= 0 {
1614        return Ok(());
1615    }
1616    let mut files = std::fs::read_dir(dir)?
1617        .filter_map(Result::ok)
1618        .filter_map(|entry| {
1619            let name = entry.file_name();
1620            let name = name.to_string_lossy();
1621            if !name.starts_with("error-") || !name.ends_with(".log") {
1622                return None;
1623            }
1624            let metadata = entry.metadata().ok()?;
1625            if metadata.is_dir() {
1626                return None;
1627            }
1628            Some((metadata.modified().ok(), entry.path()))
1629        })
1630        .collect::<Vec<_>>();
1631    files.sort_by_key(|file| std::cmp::Reverse(file.0));
1632    for (_, path) in files.into_iter().skip(max as usize) {
1633        let _ = std::fs::remove_file(path);
1634    }
1635    Ok(())
1636}
1637
1638async fn decode(raw: Vec<u8>, headers: &HeaderMap, request: bool, limit: Option<u64>) -> (Vec<u8>, Option<String>) {
1639    use async_compression::tokio::bufread::{BrotliDecoder, DeflateDecoder, GzipDecoder, ZstdDecoder};
1640    let encoding = headers
1641        .get("content-encoding")
1642        .and_then(|v| v.to_str().ok())
1643        .unwrap_or_default();
1644    let encoding = if request { encoding.trim() } else { encoding };
1645    if raw.is_empty() || encoding.is_empty() {
1646        return (raw, None);
1647    }
1648    // Go's response logger accepts one exact encoding, not stacked encodings.
1649    if !request
1650        && !matches!(
1651            encoding.to_ascii_lowercase().as_str(),
1652            "gzip" | "deflate" | "br" | "zstd"
1653        )
1654    {
1655        return (raw, None);
1656    }
1657    let mut body = raw.clone();
1658    for encoding in encoding.split(',').rev().map(|s| s.trim().to_ascii_lowercase()) {
1659        let mut reader: Pin<Box<dyn AsyncRead + Send + '_>> = match encoding.as_str() {
1660            "" | "identity" => continue,
1661            "zstd" => {
1662                let mut decoder = ZstdDecoder::new(body.as_slice());
1663                decoder.multiple_members(true);
1664                Box::pin(decoder)
1665            }
1666            "gzip" if !request => {
1667                let mut decoder = GzipDecoder::new(body.as_slice());
1668                decoder.multiple_members(true);
1669                Box::pin(decoder)
1670            }
1671            "deflate" if !request => Box::pin(DeflateDecoder::new(body.as_slice())),
1672            "br" if !request => Box::pin(BrotliDecoder::new(body.as_slice())),
1673            _ => return (raw, None),
1674        };
1675        let mut output = Vec::new();
1676        let result = if let Some(limit) = limit {
1677            reader.as_mut().take(limit + 1).read_to_end(&mut output).await
1678        } else {
1679            reader.read_to_end(&mut output).await
1680        };
1681        drop(reader);
1682        if let Err(error) = result {
1683            // ponytail: M4-0254 accepted partial: malformed Brotli can withhold
1684            // partial decoded bytes that Go accepts, and codec diagnostic text
1685            // differs. Keep the complete raw fallback; port decoder edge behavior
1686            // and Go's error classifier only if exact corrupt-stream logs are needed.
1687            return (
1688                raw,
1689                (!request).then(|| format!("failed to decompress {encoding} data: {error}")),
1690            );
1691        }
1692        if limit.is_some_and(|limit| output.len() as u64 > limit) {
1693            output.truncate(limit.unwrap() as usize);
1694            if !output.ends_with(b"\n") {
1695                output.push(b'\n');
1696            }
1697            output.extend_from_slice(b"[DECOMPRESSED REQUEST BODY TRUNCATED]");
1698            return (output, None);
1699        }
1700        body = output;
1701    }
1702    (body, None)
1703}
1704
1705#[cfg(test)]
1706mod tests {
1707    use super::*;
1708    use axum::http::{Request as HttpRequest, StatusCode};
1709    use axum::response::IntoResponse;
1710    use tower_service::Service;
1711
1712    /// `capture_go.json`, recorded with the Go generator's `buildinfo.Version` pinned to
1713    /// `cliproxy-rs-0.1.0` (tests/reference/capture/main.go). The version of this build
1714    /// replaces it, so a release bump needs no re-recording.
1715    /// A file every write to fails, on any OS (Linux-only `/dev/full` is not needed).
1716    fn unwritable(dir: &Path) -> File {
1717        let path = dir.join("unwritable");
1718        File::create(&path).unwrap();
1719        File::options().read(true).open(path).unwrap()
1720    }
1721
1722    fn capture_go() -> serde_json::Value {
1723        let text = include_str!("../tests/fixtures/capture_go.json")
1724            .replace("cliproxy-rs-0.1.0", concat!("cliproxy-rs-", env!("CARGO_PKG_VERSION")));
1725        serde_json::from_str(&text).unwrap()
1726    }
1727
1728    fn management(dir: &Path, enabled: bool, commercial: bool) -> Arc<Management> {
1729        let config = cpa_core::config::Config::parse(&format!(
1730            "observability: {{logs: {{request-log: {enabled}}}}}\nserver: {{commercial-mode: {commercial}}}\n"
1731        ))
1732        .unwrap();
1733        let rt = Arc::new(crate::testing::runtime(
1734            config,
1735            Vec::new(),
1736            cpa_exec::Executors {
1737                claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:9").unwrap(),
1738                codex: Default::default(),
1739                devices: Default::default(),
1740                openai: Default::default(),
1741                google: Default::default(),
1742            },
1743        ));
1744        Management::with_options(
1745            rt,
1746            dir.join("config.yaml"),
1747            crate::management::Options {
1748                log_dir: Some(dir.to_owned()),
1749                management_password: Some(String::new()),
1750                ..Default::default()
1751            },
1752        )
1753    }
1754
1755    fn scratch() -> PathBuf {
1756        let dir = std::env::temp_dir().join(format!("cpa-request-log-{}", uuid::Uuid::new_v4()));
1757        std::fs::create_dir_all(&dir).unwrap();
1758        dir
1759    }
1760
1761    fn canonical(raw: &str) -> String {
1762        let mut lines = raw.split('\n').map(str::to_owned).collect::<Vec<_>>();
1763        for i in 0..lines.len() {
1764            if lines[i].starts_with("Timestamp: ") {
1765                lines[i] = "Timestamp: <time>".into();
1766            }
1767            if lines[i] == "=== HEADERS ===" || lines[i] == "=== RESPONSE ===" {
1768                let mut start = i + 1;
1769                if lines.get(start).is_some_and(|s| s.starts_with("Status: ")) {
1770                    start += 1;
1771                }
1772                let end = (start..lines.len())
1773                    .find(|&j| lines[j].is_empty())
1774                    .unwrap_or(lines.len());
1775                lines[start..end].sort();
1776            }
1777        }
1778        lines.join("\n")
1779    }
1780
1781    fn contents(section: &Section) -> Vec<u8> {
1782        let mut out = Vec::new();
1783        section.write_to(&mut out).unwrap();
1784        out
1785    }
1786
1787    fn logs(dir: &Path) -> Vec<PathBuf> {
1788        std::fs::read_dir(dir)
1789            .unwrap()
1790            .flatten()
1791            .map(|e| e.path())
1792            .filter(|p| p.extension().is_some_and(|e| e == "log"))
1793            .collect()
1794    }
1795
1796    #[tokio::test]
1797    async fn upstream_contract_matches_real_go() {
1798        use cpa_core::exec::{CaptureEvent::*, UpstreamRequest};
1799        assert!(!cpa_core::exec::CaptureSink::default().enabled());
1800        assert!(!cpa_core::exec::CaptureSink::default().logs_responses());
1801        let fixture: serde_json::Value =
1802            serde_json::from_str(include_str!("../tests/fixtures/upstream_capture_go.json")).unwrap();
1803        for case in fixture.as_array().unwrap() {
1804            let case = case.clone();
1805            let dir = scratch();
1806            let state = management(&dir, case["kind"] != "disabled" && case["kind"] != "reload", false);
1807            let rt = state.rt.clone();
1808            let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
1809                let case = case.clone();
1810                let rt = rt.clone();
1811                async move {
1812                    let sink = log.capture_sink();
1813                    let headers = [
1814                        ("Authorization".into(), "Bearer 1234567890".into()),
1815                        ("X-Token".into(), "abcdefghi".into()),
1816                        ("Z-Header".into(), "first".into()),
1817                        ("Z-Header".into(), "second".into()),
1818                    ];
1819                    let response_headers = [("X-Secret".into(), "1234567890".into())];
1820                    let info = |oauth, empty| UpstreamRequest {
1821                        url: "https://fixture.invalid/private",
1822                        method: "POST",
1823                        headers: &headers,
1824                        body: if empty { b"" } else { b"request body" },
1825                        provider: " codex ",
1826                        auth_id: " fixture-id ",
1827                        auth_label: " fixture-label ",
1828                        auth_type: if oauth { "oauth" } else { " API_KEY " },
1829                        auth_value: if oauth { "never-print-oauth" } else { " 1234567890 " },
1830                    };
1831                    let kind = case["kind"].as_str().unwrap();
1832                    // Executors skip buffering response-only log data on this signal.
1833                    let logging = !matches!(kind, "disabled" | "reload");
1834                    assert_eq!(sink.logs_responses(), logging, "{kind}: responses logged");
1835                    if !matches!(kind, "missing" | "websocket") {
1836                        sink.record(Request(info(kind == "oauth", false)));
1837                    }
1838                    if kind == "websocket" {
1839                        sink.record(WebsocketRequest(info(false, false)));
1840                        sink.record(WebsocketHandshake(101, &response_headers));
1841                        sink.record(WebsocketResponse(b"  {\"type\":\"response.created\"} \n"));
1842                        sink.record(WebsocketError {
1843                            stage: " read ",
1844                            error: "fixture disconnect",
1845                        });
1846                    } else {
1847                        if kind == "reload" {
1848                            rt.publish_config(
1849                                cpa_core::config::Config::parse("observability: {logs: {request-log: true}}\n")
1850                                    .unwrap(),
1851                            );
1852                            assert!(sink.logs_responses(), "reload: responses logged");
1853                        }
1854                        sink.record(ResponseMetadata(201, &response_headers));
1855                        sink.record(ResponseMetadata(202, &[("Ignored".into(), "yes".into())]));
1856                        sink.record(ResponseChunk(b"  event: first\n"));
1857                        sink.record(ResponseChunk(b"data: {\"a\":1}\n"));
1858                        sink.record(ResponseChunk(b"\n data: second \n"));
1859                        sink.record(ResponseError("fixture error"));
1860                        sink.record(ResponseError("second error"));
1861                        if kind == "http" {
1862                            sink.record(Request(info(true, true)));
1863                            sink.record(ResponseError("failed before response"));
1864                        }
1865                    }
1866                    let locked = log.0.lock().unwrap();
1867                    let record = locked.as_ref().unwrap();
1868                    for (name, bytes) in [
1869                        (
1870                            "request",
1871                            if kind == "disabled" {
1872                                record.deferred_api_request.concat()
1873                            } else {
1874                                contents(&record.api_request)
1875                            },
1876                        ),
1877                        ("response", contents(&record.api_response)),
1878                        ("timeline", contents(&record.api_timeline)),
1879                    ] {
1880                        assert_eq!(
1881                            canonical(std::str::from_utf8(&bytes).unwrap()).trim_end_matches('\n'),
1882                            case[name].as_str().unwrap().trim_end_matches('\n'),
1883                            "{kind} {name}"
1884                        );
1885                        assert!(!String::from_utf8_lossy(&bytes).contains("never-print-oauth"));
1886                    }
1887                    "ok"
1888                }
1889            });
1890            let mut app = router(&state, app);
1891            let response = app
1892                .call(
1893                    HttpRequest::builder()
1894                        .method("POST")
1895                        .uri("/fixture")
1896                        .body(Body::empty())
1897                        .unwrap(),
1898                )
1899                .await
1900                .unwrap();
1901            axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
1902            tokio::time::sleep(std::time::Duration::from_millis(30)).await;
1903            std::fs::remove_dir_all(dir).unwrap();
1904        }
1905    }
1906
1907    struct HomeLogs {
1908        available: bool,
1909        logs: Mutex<Vec<Vec<u8>>>,
1910        gate: Option<Arc<(tokio::sync::Notify, tokio::sync::Notify)>>,
1911    }
1912    impl crate::remote::RemoteDispatch for HomeLogs {
1913        fn available(&self) -> bool {
1914            self.available
1915        }
1916        fn dispatch(
1917            &self,
1918            _: crate::remote::RemoteRequest,
1919        ) -> futures_util::future::BoxFuture<'_, Result<crate::remote::RemoteGrant, crate::remote::RemoteError>>
1920        {
1921            Box::pin(async { panic!("no dispatch") })
1922        }
1923        fn models(
1924            &self,
1925            _: Vec<(String, String)>,
1926            _: Vec<(String, String)>,
1927        ) -> futures_util::future::BoxFuture<'_, Result<Vec<u8>, crate::remote::ModelsError>> {
1928            Box::pin(async { Err(crate::remote::ModelsError::Unavailable) })
1929        }
1930        fn request_log(&self, payload: Vec<u8>) -> futures_util::future::BoxFuture<'_, Result<(), String>> {
1931            Box::pin(async move {
1932                if let Some(gate) = &self.gate {
1933                    gate.0.notify_one();
1934                    gate.1.notified().await;
1935                }
1936                self.logs.lock().unwrap().push(payload);
1937                Ok(())
1938            })
1939        }
1940    }
1941
1942    #[tokio::test]
1943    async fn home_forwards_full_logs_but_forced_errors_remain_local() {
1944        for (enabled, available) in [(true, true), (true, false), (false, true)] {
1945            let dir = scratch();
1946            let state = management(&dir, enabled, false);
1947            let home = Arc::new(HomeLogs {
1948                available,
1949                logs: Mutex::new(Vec::new()),
1950                gate: None,
1951            });
1952            state.rt.set_remote_dispatch(Some(home.clone()));
1953            let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1954            let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
1955                let _ = tx.send(log);
1956                async { (StatusCode::INTERNAL_SERVER_ERROR, "fixture error") }
1957            });
1958            let mut app = router(&state, app);
1959            let mut request = HttpRequest::builder()
1960                .method("POST")
1961                .uri("/fixture")
1962                .header("authorization", "Bearer 1234567890")
1963                .body(Body::empty())
1964                .unwrap();
1965            request.extensions_mut().insert(RequestId("fixture-request-id".into()));
1966            let response = app.call(request).await.unwrap();
1967            axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
1968            // The handler ran inside `app.call`, so its log is already queued.
1969            let log = rx.try_recv().expect("handler sent its RequestLog");
1970            // The log is finalized in a spawned task after the body ends: forwarded to
1971            // Home, or written on the blocking pool. Wait for that task, not a fixed time,
1972            // so the counts below are final.
1973            let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
1974            while !log.1.done.load(Ordering::Acquire) {
1975                assert!(tokio::time::Instant::now() < deadline, "request log never finalized");
1976                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1977            }
1978            let captured = home.logs.lock().unwrap();
1979            assert_eq!(captured.len(), usize::from(enabled && available));
1980            assert_eq!(logs(&dir).len(), usize::from(!enabled));
1981            if let Some(payload) = captured.first() {
1982                let payload: serde_json::Value = serde_json::from_slice(payload).unwrap();
1983                assert_eq!(payload["request_id"], "fixture-request-id");
1984                assert_eq!(payload["headers"]["Authorization"][0], "Bearer 1234567890");
1985                let content = payload["request_log"].as_str().unwrap();
1986                assert!(content.contains("Authorization: Bearer 1234...7890"));
1987                assert!(content.contains(
1988                    "=== RESPONSE ===\nStatus: 500\nContent-Type: text/plain; charset=utf-8\n\nfixture error\n"
1989                ));
1990            }
1991            std::fs::remove_dir_all(dir).unwrap();
1992        }
1993    }
1994
1995    #[tokio::test]
1996    async fn served_composition_captures_before_head_stripping_and_length_inference() {
1997        let dir = scratch();
1998        let state = management(&dir, true, false);
1999        let rest = axum::Router::new().route(
2000            "/fixture",
2001            axum::routing::post(|| async { ([("content-type", "application/json")], "outbound\n") }).head(|| async {
2002                (
2003                    StatusCode::INTERNAL_SERVER_ERROR,
2004                    [("content-length", "9")],
2005                    "outbound\n",
2006                )
2007            }),
2008        );
2009        let mut app = router(
2010            &state,
2011            crate::app(state.rt.clone(), rest).layer(axum::middleware::from_fn_with_state(
2012                state.rt.clone(),
2013                crate::management::cors,
2014            )),
2015        );
2016        for method in ["POST", "HEAD"] {
2017            let response = app
2018                .call(
2019                    HttpRequest::builder()
2020                        .method(method)
2021                        .uri("/fixture")
2022                        // No client keys: CORS goes to loopback origins only.
2023                        .header("origin", "http://localhost:5173")
2024                        .body(Body::empty())
2025                        .unwrap(),
2026                )
2027                .await
2028                .unwrap();
2029            let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
2030            assert_eq!(body, if method == "HEAD" { "" } else { "outbound\n" });
2031        }
2032        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
2033        loop {
2034            let contents: Vec<_> = logs(&dir).iter().map(|p| std::fs::read_to_string(p).unwrap()).collect();
2035            if contents.len() == 2 && contents.iter().all(|s| s.ends_with("outbound\n\n")) {
2036                for content in contents {
2037                    assert!(content.contains("Access-Control-Allow-Origin: http://localhost:5173"));
2038                    assert_eq!(content.contains("Content-Length: 9"), content.contains("Method: HEAD"));
2039                }
2040                break;
2041            }
2042            assert!(
2043                tokio::time::Instant::now() < deadline,
2044                "missing pre-framing capture: {contents:?}"
2045            );
2046            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2047        }
2048        std::fs::remove_dir_all(dir).unwrap();
2049    }
2050
2051    #[tokio::test]
2052    async fn cancelling_websocket_close_does_not_cancel_final_delivery() {
2053        let dir = scratch();
2054        let state = management(&dir, true, false);
2055        let gate = Arc::new((tokio::sync::Notify::new(), tokio::sync::Notify::new()));
2056        let home = Arc::new(HomeLogs {
2057            available: true,
2058            logs: Mutex::new(Vec::new()),
2059            gate: Some(gate.clone()),
2060        });
2061        state.rt.set_remote_dispatch(Some(home.clone()));
2062        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
2063        let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
2064            let tx = tx.clone();
2065            async move {
2066                tx.send((log.clone(), log.detach_websocket().unwrap())).await.unwrap();
2067                StatusCode::SWITCHING_PROTOCOLS
2068            }
2069        });
2070        let mut app = router(&state, app);
2071        let response = app
2072            .call(
2073                HttpRequest::builder()
2074                    .uri("/v1/responses")
2075                    .header("upgrade", "websocket")
2076                    .body(Body::empty())
2077                    .unwrap(),
2078            )
2079            .await
2080            .unwrap();
2081        drop(response);
2082        let (log, socket) = rx.recv().await.unwrap();
2083        socket
2084            .append_part(b"Timestamp: fixture\nEvent: websocket.disconnect\nbye\n")
2085            .unwrap();
2086        let caller = tokio::spawn(socket.close());
2087        gate.0.notified().await;
2088        caller.abort();
2089        gate.1.notify_one();
2090        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2091            while !log.1.done.load(Ordering::Acquire) {
2092                tokio::task::yield_now().await;
2093            }
2094        })
2095        .await
2096        .unwrap();
2097        assert_eq!(home.logs.lock().unwrap().len(), 1);
2098        assert!(logs(&dir).is_empty());
2099        std::fs::remove_dir_all(dir).unwrap();
2100    }
2101
2102    #[tokio::test]
2103    async fn go_middleware_goldens() {
2104        let fixture: serde_json::Value = capture_go();
2105        let date = regex::Regex::new(r"\d{4}-\d\d-\d\dT\d{6}").unwrap();
2106        for case in fixture["cases"].as_array().unwrap() {
2107            let dir = scratch();
2108            let state = management(&dir, case["enabled"].as_bool().unwrap(), false);
2109            let status = StatusCode::from_u16(case["status"].as_u64().unwrap() as u16).unwrap();
2110            let consume = case["consume"].as_i64().unwrap();
2111            let streaming = case["stream"].as_bool().unwrap();
2112            let app = axum::Router::new().fallback(move |request: Request| async move {
2113                if consume < 0 {
2114                    let _ = axum::body::to_bytes(request.into_body(), usize::MAX).await.unwrap();
2115                } else {
2116                    use futures_util::StreamExt;
2117                    let mut body = request.into_body().into_data_stream();
2118                    assert_eq!(body.next().await.unwrap().unwrap().len(), consume as usize);
2119                    // Do not drain the unread rest, just like the Go fixture.
2120                }
2121                let mut response = Response::new(Body::from("outbound\n"));
2122                *response.status_mut() = status;
2123                response.headers_mut().insert(
2124                    "content-type",
2125                    if streaming {
2126                        "text/event-stream"
2127                    } else {
2128                        "application/json"
2129                    }
2130                    .parse()
2131                    .unwrap(),
2132                );
2133                response
2134            });
2135            let mut app = router(&state, app);
2136            let length = case["body_length"].as_u64().unwrap() as usize;
2137            let body = if consume >= 0 {
2138                Body::from_stream(futures_util::stream::iter([
2139                    Ok::<_, io::Error>(Bytes::from(vec![b'x'; consume as usize])),
2140                    Ok(Bytes::from(vec![b'x'; length - consume as usize])),
2141                ]))
2142            } else {
2143                Body::from(case["body"].as_str().unwrap().to_owned())
2144            };
2145            let mut request = HttpRequest::builder()
2146                .method(case["method"].as_str().unwrap())
2147                .uri(case["path"].as_str().unwrap())
2148                .header("authorization", "Bearer 1234567890")
2149                .header("content-length", length.to_string())
2150                .body(body)
2151                .unwrap();
2152            request.extensions_mut().insert(RequestId("0198-aaaa-a1b2c3d4".into()));
2153            let response = app.call(request).await.unwrap();
2154            assert_eq!(response.status(), status);
2155            let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
2156            assert_eq!(body, if case["method"] == "HEAD" { "" } else { "outbound\n" });
2157            let expected = case["content"].as_str().unwrap();
2158            let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
2159            loop {
2160                let files = logs(&dir);
2161                let actual = files
2162                    .first()
2163                    .and_then(|p| std::fs::read_to_string(p).ok())
2164                    .unwrap_or_default();
2165                if canonical(&actual) == expected {
2166                    break;
2167                }
2168                assert!(tokio::time::Instant::now() < deadline, "case {case}: got {actual}");
2169                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2170            }
2171            // Wait for scheduled cleanup too, so discarded error-only capture
2172            // cannot leave files behind and an unexpected later write fails.
2173            tokio::time::sleep(std::time::Duration::from_millis(30)).await;
2174            let files = logs(&dir);
2175            assert_eq!(files.len(), usize::from(!expected.is_empty()));
2176            if let Some(file) = files.first() {
2177                assert_eq!(
2178                    date.replace_all(file.file_name().unwrap().to_str().unwrap(), "<date>"),
2179                    case["filename"].as_str().unwrap()
2180                );
2181            }
2182            assert!(
2183                std::fs::read_dir(&dir)
2184                    .unwrap()
2185                    .flatten()
2186                    .all(|e| e.path().extension().is_none_or(|e| e != "tmp"))
2187            );
2188            std::fs::remove_dir_all(dir).unwrap();
2189        }
2190    }
2191
2192    #[tokio::test]
2193    async fn websocket_sink_stays_open_after_upgrade_and_matches_go() {
2194        use base64::Engine;
2195        let fixture: serde_json::Value = capture_go();
2196        let dir = scratch();
2197        let state = management(&dir, true, false);
2198        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
2199        let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
2200            let tx = tx.clone();
2201            async move {
2202                assert_eq!(log.request_id(), "0198-aaaa-a1b2c3d4");
2203                tx.send(log.detach_websocket().unwrap()).await.unwrap();
2204                StatusCode::SWITCHING_PROTOCOLS.into_response()
2205            }
2206        });
2207        let mut app = router(&state, app);
2208        let mut request = HttpRequest::builder()
2209            .uri("/v1/responses")
2210            .header("upgrade", "websocket")
2211            .body(Body::empty())
2212            .unwrap();
2213        request.extensions_mut().insert(RequestId("0198-aaaa-a1b2c3d4".into()));
2214        let response = app.call(request).await.unwrap();
2215        drop(response);
2216        let sink = rx.recv().await.unwrap();
2217        tokio::time::sleep(std::time::Duration::from_millis(30)).await;
2218        assert!(logs(&dir).is_empty(), "HTTP 101 must not flush the transcript");
2219        for part in fixture["websocket"]["parts"].as_array().unwrap() {
2220            sink.append_part(
2221                &base64::engine::general_purpose::STANDARD
2222                    .decode(part.as_str().unwrap())
2223                    .unwrap(),
2224            )
2225            .unwrap();
2226        }
2227        sink.close().await;
2228        let files = logs(&dir);
2229        assert_eq!(files.len(), 1);
2230        let content = std::fs::read_to_string(&files[0]).unwrap();
2231        assert_eq!(canonical(&content), fixture["websocket"]["content"]);
2232        assert!(!content.contains("REQUEST BODY") && !content.contains("=== RESPONSE ==="));
2233        std::fs::remove_dir_all(dir).unwrap();
2234    }
2235
2236    #[test]
2237    fn go_masking_collisions_and_retention() {
2238        let fixture: serde_json::Value = capture_go();
2239        for case in fixture["masks"].as_array().unwrap() {
2240            let name = case["name"].as_str().unwrap().to_ascii_lowercase();
2241            let value = case["value"].as_str().unwrap();
2242            let mut want = case["out"].as_str().unwrap().to_owned();
2243            // Deliberate difference: Go leaves a masked two-byte value in the clear.
2244            if value == "ab" && masked_header(&name, b"abc") != b"abc" {
2245                want = "...".into();
2246            }
2247            // Deliberate difference: Go logs cookie values unchanged.
2248            if cookie(&name) {
2249                want = REDACTED.into();
2250            }
2251            assert_eq!(String::from_utf8(masked_header(&name, value.as_bytes())).unwrap(), want);
2252        }
2253        let dir = scratch();
2254        let name = "error-v1-responses-2026-10-03T040506-a1b2c3d4.log";
2255        unique_file(&dir, name).unwrap().write_all(b"original").unwrap();
2256        unique_file(&dir, name).unwrap().write_all(b"second").unwrap();
2257        assert_eq!(std::fs::read(dir.join(name)).unwrap(), b"original");
2258        let second = dir.join("error-v1-responses-2026-10-03T040506_1-a1b2c3d4.log");
2259        assert_eq!(std::fs::read(&second).unwrap(), b"second");
2260        File::options()
2261            .write(true)
2262            .open(&second)
2263            .unwrap()
2264            .set_modified(std::time::SystemTime::UNIX_EPOCH)
2265            .unwrap();
2266        std::fs::write(dir.join("main.log"), b"protected").unwrap();
2267        retain_errors(&dir, 0).unwrap();
2268        assert_eq!(logs(&dir).len(), 3);
2269        retain_errors(&dir, 1).unwrap();
2270        assert!(!second.exists());
2271        assert!(dir.join(name).exists() && dir.join("main.log").exists());
2272        let at = DateTime::parse_from_rfc3339("2026-10-03T04:05:06.12001Z")
2273            .unwrap()
2274            .with_timezone(&Local);
2275        assert!(timestamp(at).contains(".12001"));
2276        assert_eq!(
2277            filename("/v1//x:  y?secret=not-in-name", "0198-aaaa-a1b2c3d4", at, true),
2278            format!("error-v1-x-y-{}-a1b2c3d4.log", at.format("%Y-%m-%dT%H%M%S"))
2279        );
2280        std::fs::remove_dir_all(dir).unwrap();
2281    }
2282
2283    #[tokio::test]
2284    async fn commercial_mode_disables_only_request_capture_and_is_startup_only() {
2285        for commercial in [false, true] {
2286            let dir = scratch();
2287            let state = management(&dir, true, commercial);
2288            let app = axum::Router::new().fallback(move |request: Request| async move {
2289                assert_eq!(request.extensions().get::<RequestLog>().is_none(), commercial);
2290                StatusCode::BAD_REQUEST
2291            });
2292            let mut app = router(&state, app);
2293            let new = cpa_core::config::Config::parse(&format!(
2294                "server: {{commercial-mode: {}}}\nobservability: {{logs: {{request-log: true}}}}\n",
2295                !commercial
2296            ))
2297            .unwrap();
2298            state.rt.publish_config(new);
2299            let response = app
2300                .call(
2301                    HttpRequest::builder()
2302                        .method("POST")
2303                        .uri("/v1/responses")
2304                        .body(Body::empty())
2305                        .unwrap(),
2306                )
2307                .await
2308                .unwrap();
2309            assert_eq!(response.status(), StatusCode::BAD_REQUEST);
2310            drop(response);
2311            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2312            assert_eq!(logs(&dir).len(), usize::from(!commercial));
2313            std::fs::remove_dir_all(dir).unwrap();
2314        }
2315    }
2316
2317    /// Claude Code's start-up `HEAD /api/hello` must not write a forced error log: with
2318    /// the default `error-logs-max-files: 10` those probes pushed real failures out (#262).
2319    /// An unregistered HEAD still writes one, as in Go.
2320    #[tokio::test]
2321    async fn claude_code_hello_probe_writes_no_error_log() {
2322        let dir = scratch();
2323        let state = management(&dir, false, false);
2324        let mut app = router(&state, crate::router(state.rt.clone()));
2325        for (path, status) in [("/api/hello", StatusCode::OK), ("/api/nothing", StatusCode::NOT_FOUND)] {
2326            let response = app
2327                .call(
2328                    HttpRequest::builder()
2329                        .method("HEAD")
2330                        .uri(path)
2331                        .body(Body::empty())
2332                        .unwrap(),
2333                )
2334                .await
2335                .unwrap();
2336            assert_eq!(response.status(), status, "{path}");
2337            assert!(
2338                axum::body::to_bytes(response.into_body(), usize::MAX)
2339                    .await
2340                    .unwrap()
2341                    .is_empty()
2342            );
2343        }
2344        // The control's log shows the forced capture ran; give the probe's the same time.
2345        for _ in 0..100 {
2346            if !logs(&dir).is_empty() {
2347                break;
2348            }
2349            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2350        }
2351        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2352        let names = logs(&dir)
2353            .iter()
2354            .map(|p| p.file_name().unwrap().to_string_lossy().into_owned())
2355            .collect::<Vec<_>>();
2356        assert_eq!(names.len(), 1, "{names:?}");
2357        assert!(names[0].starts_with("error-api-nothing-"), "{names:?}");
2358        std::fs::remove_dir_all(dir).unwrap();
2359    }
2360
2361    #[tokio::test]
2362    async fn capture_preserves_transport_errors_and_classifies_cancellation() {
2363        let dir = scratch();
2364        let state = management(&dir, true, false);
2365        let app = axum::Router::new().fallback(|request: Request| async move {
2366            assert!(axum::body::to_bytes(request.into_body(), usize::MAX).await.is_err());
2367            StatusCode::BAD_REQUEST
2368        });
2369        let mut app = router(&state, app);
2370        let body = Body::from_stream(futures_util::stream::iter([
2371            Ok(Bytes::from_static(b"partial")),
2372            Err(io::Error::other("broken transport")),
2373        ]));
2374        let response = app
2375            .call(
2376                HttpRequest::builder()
2377                    .method("POST")
2378                    .uri("/v1/responses")
2379                    .body(body)
2380                    .unwrap(),
2381            )
2382            .await
2383            .unwrap();
2384        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
2385        drop(response);
2386        assert!(logs(&dir).is_empty(), "capture failure bypasses logging as in Go");
2387        std::fs::remove_dir_all(dir).unwrap();
2388
2389        for (status, error, expected) in [
2390            (500, "context CANCELED", false),
2391            (500, "client closed request", false),
2392            (499, "gone", false),
2393            (502, "upstream failure", true),
2394        ] {
2395            let dir = scratch();
2396            let state = management(&dir, false, false);
2397            let app =
2398                axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| async move {
2399                    assert!(
2400                        log.detach_websocket().is_none(),
2401                        "disabled capture has no timeline work"
2402                    );
2403                    log.api_error(status, error.into());
2404                    StatusCode::OK
2405                });
2406            let mut app = router(&state, app);
2407            let response = app
2408                .call(
2409                    HttpRequest::builder()
2410                        .method("POST")
2411                        .uri("/v1/responses")
2412                        .body(Body::empty())
2413                        .unwrap(),
2414                )
2415                .await
2416                .unwrap();
2417            drop(response);
2418            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2419            assert_eq!(logs(&dir).len(), usize::from(expected));
2420            std::fs::remove_dir_all(dir).unwrap();
2421        }
2422    }
2423
2424    #[tokio::test]
2425    async fn detached_close_waits_for_http_metadata_and_drop_finalizes_once() {
2426        for cancel in [false, true] {
2427            let dir = scratch();
2428            let state = management(&dir, true, false);
2429            let (tx, mut rx) = tokio::sync::mpsc::channel(1);
2430            let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
2431                let tx = tx.clone();
2432                async move {
2433                    let sink = log.detach_websocket().unwrap();
2434                    assert!(log.detach_websocket().is_none(), "only one connection owner");
2435                    if cancel {
2436                        drop(sink);
2437                        tx.send(None).await.unwrap();
2438                        std::future::pending::<()>().await;
2439                    } else {
2440                        let close = tokio::spawn(sink.close());
2441                        tokio::task::yield_now().await;
2442                        assert!(!close.is_finished(), "must wait for HTTP status");
2443                        tx.send(Some(close)).await.unwrap();
2444                    }
2445                    StatusCode::BAD_REQUEST
2446                }
2447            });
2448            let mut app = router(&state, app);
2449            let request = HttpRequest::builder()
2450                .method("POST")
2451                .uri("/v1/responses")
2452                .body(Body::empty())
2453                .unwrap();
2454            if cancel {
2455                let mut response = Box::pin(app.call(request));
2456                assert!(futures_util::poll!(&mut response).is_pending());
2457                rx.recv().await.unwrap();
2458                drop(response);
2459                tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2460            } else {
2461                let response = app.call(request).await.unwrap();
2462                let close = rx.recv().await.unwrap().unwrap();
2463                tokio::time::timeout(std::time::Duration::from_secs(5), close)
2464                    .await
2465                    .unwrap()
2466                    .unwrap();
2467                drop(response);
2468            }
2469            let files = logs(&dir);
2470            assert_eq!(files.len(), 1);
2471            let content = std::fs::read_to_string(&files[0]).unwrap();
2472            assert!(content.contains(if cancel { "Status: 200\n" } else { "Status: 400\n" }));
2473            std::fs::remove_dir_all(dir).unwrap();
2474        }
2475    }
2476
2477    #[tokio::test]
2478    async fn deferred_body_boundaries_are_transparent_and_disk_errors_are_sticky() {
2479        for size in [32 << 20, (32 << 20) + 1] {
2480            let dir = scratch();
2481            let capture = Arc::new(Mutex::new(Deferred {
2482                spool: Spool::new(&dir, "request-body").unwrap(),
2483                length: None,
2484                read: 0,
2485                captured: 0,
2486                eof: false,
2487                truncated: false,
2488                failed: false,
2489            }));
2490            let body = Body::new(RequestBody {
2491                body: Body::from(vec![b'x'; size]),
2492                deferred: capture.clone(),
2493            });
2494            assert_eq!(axum::body::to_bytes(body, usize::MAX).await.unwrap().len(), size);
2495            let (bytes, marker) = capture.lock().unwrap().bytes().unwrap();
2496            assert_eq!(bytes.len(), 32 << 20);
2497            assert_eq!(
2498                marker,
2499                if size == 32 << 20 {
2500                    ""
2501                } else {
2502                    "[REQUEST BODY TRUNCATED: captured first 33554432 bytes]"
2503                }
2504            );
2505            drop(capture);
2506            assert!(std::fs::read_dir(&dir).unwrap().next().is_none());
2507            std::fs::remove_dir_all(dir).unwrap();
2508        }
2509        let dir = scratch();
2510        let mut spool = Spool::new(&dir, "request-body").unwrap();
2511        spool.file = unwritable(&dir);
2512        let capture = Arc::new(Mutex::new(Deferred {
2513            spool,
2514            length: None,
2515            read: 0,
2516            captured: 0,
2517            eof: false,
2518            truncated: false,
2519            failed: false,
2520        }));
2521        let body = Body::new(RequestBody {
2522            body: Body::from("still forwarded"),
2523            deferred: capture.clone(),
2524        });
2525        assert_eq!(axum::body::to_bytes(body, usize::MAX).await.unwrap(), "still forwarded");
2526        assert!(capture.lock().unwrap().bytes().is_err());
2527        assert_eq!(capture.lock().unwrap().captured, 0);
2528        drop(capture);
2529        std::fs::remove_dir_all(dir).unwrap();
2530    }
2531
2532    #[tokio::test]
2533    async fn response_disk_failure_does_not_finalize_early_and_queue_is_nonblocking() {
2534        let dir = scratch();
2535        let state = management(&dir, true, false);
2536        let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
2537        let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
2538            let sender = sender.clone();
2539            async move {
2540                sender.send(log).await.unwrap();
2541                Body::from_stream(futures_util::stream::pending::<io::Result<Bytes>>())
2542            }
2543        });
2544        let mut app = router(&state, app);
2545        let response = app
2546            .call(
2547                HttpRequest::builder()
2548                    .method("POST")
2549                    .uri("/v1/responses")
2550                    .body(Body::empty())
2551                    .unwrap(),
2552            )
2553            .await
2554            .unwrap();
2555        let sink = receiver.recv().await.unwrap();
2556        let (tx, rx) = tokio::sync::mpsc::channel(100);
2557        let body = ResponseBody {
2558            body: Body::from_stream(futures_util::stream::iter(
2559                (0..102).map(|_| Ok::<_, io::Error>(Bytes::from_static(b"x"))),
2560            )),
2561            sink: sink.clone(),
2562            guard: Completion(None),
2563            chunks: Some(tx.clone()),
2564            buffer: false,
2565        };
2566        let forwarded = tokio::time::timeout(
2567            std::time::Duration::from_secs(1),
2568            axum::body::to_bytes(Body::new(body), usize::MAX),
2569        )
2570        .await
2571        .unwrap()
2572        .unwrap();
2573        assert_eq!(forwarded.len(), 102, "stalled logger must not delay/drop client output");
2574        assert_eq!(rx.len(), 100, "logger alone drops chunks above its queue limit");
2575        let file = unwritable(&dir);
2576        let worker = tokio::spawn(spool_response(file, rx, sink.clone()));
2577        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2578        assert!(!worker.is_finished(), "disk failure must wait for HTTP completion");
2579        assert!(sink.0.lock().unwrap().is_some());
2580        assert!(logs(&dir).is_empty());
2581        drop(tx);
2582        worker.await.unwrap();
2583        assert!(sink.0.lock().unwrap().is_none());
2584        assert!(
2585            logs(&dir).is_empty(),
2586            "failed stream is not published as a complete log"
2587        );
2588        drop(response);
2589        std::fs::remove_dir_all(dir).unwrap();
2590    }
2591
2592    #[tokio::test]
2593    async fn hot_enable_captures_only_frames_written_while_enabled_like_go() {
2594        use futures_util::StreamExt;
2595        let fixture: serde_json::Value = capture_go();
2596        for case in fixture["reloads"].as_array().unwrap() {
2597            let dir = scratch();
2598            let state = management(&dir, false, false);
2599            let app = axum::Router::new().fallback(|| async {
2600                Response::builder()
2601                    .header("content-type", "application/json")
2602                    .body(Body::from_stream(futures_util::stream::iter([
2603                        Ok::<_, io::Error>(Bytes::from_static(b"first\n")),
2604                        Ok(Bytes::from_static(b"second\n")),
2605                    ])))
2606                    .unwrap()
2607            });
2608            let mut app = router(&state, app);
2609            let response = app
2610                .call(
2611                    HttpRequest::builder()
2612                        .method("POST")
2613                        .uri("/v1/responses")
2614                        .body(Body::empty())
2615                        .unwrap(),
2616                )
2617                .await
2618                .unwrap();
2619            let enable = || {
2620                state.rt.publish_config(
2621                    cpa_core::config::Config::parse("observability: {logs: {request-log: true}}\n").unwrap(),
2622                )
2623            };
2624            if case["before_first"] == true {
2625                enable();
2626            }
2627            let mut body = response.into_body().into_data_stream();
2628            assert_eq!(body.next().await.unwrap().unwrap(), "first\n");
2629            enable();
2630            assert_eq!(body.next().await.unwrap().unwrap(), "second\n");
2631            assert!(body.next().await.is_none());
2632            let expected = case["content"].as_str().unwrap();
2633            let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
2634            loop {
2635                let actual = logs(&dir)
2636                    .first()
2637                    .and_then(|p| std::fs::read_to_string(p).ok())
2638                    .unwrap_or_default();
2639                if canonical(&actual) == expected {
2640                    break;
2641                }
2642                assert!(
2643                    tokio::time::Instant::now() < deadline,
2644                    "got {actual}, expected {expected}"
2645                );
2646                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2647            }
2648            std::fs::remove_dir_all(dir).unwrap();
2649        }
2650    }
2651
2652    #[tokio::test]
2653    async fn compression_body_bytes_match_real_go_including_raw_error_fallback() {
2654        use base64::Engine;
2655        let fixture: serde_json::Value = capture_go();
2656        for case in fixture["brotli_prefixes"].as_array().unwrap() {
2657            let input = base64::engine::general_purpose::STANDARD
2658                .decode(case["input"].as_str().unwrap())
2659                .unwrap();
2660            let mut headers = HeaderMap::new();
2661            headers.insert("content-encoding", "br".parse().unwrap());
2662            let (output, error) = decode(input.clone(), &headers, false, None).await;
2663            if error.is_some() {
2664                assert_eq!(output, input, "never discard bytes on codec failure");
2665            } else {
2666                assert!(!case["error"].as_bool().unwrap());
2667                assert_eq!(
2668                    output,
2669                    base64::engine::general_purpose::STANDARD
2670                        .decode(case["output"].as_str().unwrap())
2671                        .unwrap()
2672                );
2673            }
2674        }
2675        for case in fixture["compression"].as_array().unwrap() {
2676            let raw = base64::engine::general_purpose::STANDARD
2677                .decode(case["input"].as_str().unwrap())
2678                .unwrap();
2679            let log = base64::engine::general_purpose::STANDARD
2680                .decode(case["raw"].as_str().unwrap())
2681                .unwrap();
2682            let section = b"=== RESPONSE ===\n";
2683            let start = log.windows(section.len()).position(|b| b == section).unwrap() + section.len();
2684            let start = start + log[start..].windows(2).position(|b| b == b"\n\n").unwrap() + 2;
2685            let logged_body = &log[start..log.len() - 1];
2686            let mut headers = HeaderMap::new();
2687            headers.insert("content-encoding", case["encoding"].as_str().unwrap().parse().unwrap());
2688            let (body, error) = decode(raw.clone(), &headers, false, None).await;
2689            if logged_body
2690                .windows(b"\n[DECOMPRESSION ERROR: ".len())
2691                .any(|bytes| bytes == b"\n[DECOMPRESSION ERROR: ")
2692            {
2693                assert_eq!(body, raw);
2694                assert!(logged_body.starts_with(&body));
2695                assert!(logged_body[body.len()..].starts_with(b"\n[DECOMPRESSION ERROR: "));
2696                assert!(error.is_some());
2697                // Exact malformed-codec diagnostic wording remains an explicit
2698                // parity gap; raw bytes and annotation presence are verified here.
2699            } else if case["invalid"] == true && case["encoding"] == "br" {
2700                // Accepted corrupt-Brotli difference: Go permits partial output;
2701                // Rust reports failure and preserves the complete raw input.
2702                assert_eq!(body, raw);
2703                assert!(error.is_some());
2704            } else {
2705                assert_eq!(body, logged_body);
2706                assert!(error.is_none());
2707            }
2708            if case["encoding"] == "zstd" {
2709                let (request_body, error) = decode(raw.clone(), &headers, true, None).await;
2710                assert_eq!(request_body, if case["invalid"] == true { raw } else { body });
2711                assert!(error.is_none(), "Go silently retains undecodable inbound bodies");
2712            }
2713        }
2714    }
2715
2716    /// Serves one request (sending a Cookie) whose handler feeds `events` to the
2717    /// capture sink and answers `status` with a Set-Cookie, then returns the
2718    /// persisted log file.
2719    async fn persisted(
2720        enabled: bool,
2721        status: StatusCode,
2722        events: impl Fn(&RequestLog) + Clone + Send + Sync + 'static,
2723    ) -> Vec<u8> {
2724        let dir = scratch();
2725        let state = management(&dir, enabled, false);
2726        let app = axum::Router::new().fallback(move |axum::Extension(log): axum::Extension<RequestLog>| {
2727            let events = events.clone();
2728            async move {
2729                events(&log);
2730                (status, [("set-cookie", "down=c00kie-down-set")], "done")
2731            }
2732        });
2733        let mut app = router(&state, app);
2734        let request = HttpRequest::builder()
2735            .method("POST")
2736            .uri("/fixture")
2737            .header("cookie", "client=c00kie-client")
2738            .body(Body::empty())
2739            .unwrap();
2740        let response = app.call(request).await.unwrap();
2741        axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
2742        let mut content = Vec::new();
2743        for _ in 0..500 {
2744            if let [log] = &logs(&dir)[..] {
2745                content = std::fs::read(log).unwrap();
2746                if content.trim_ascii_end().ends_with(b"\ndone") {
2747                    break;
2748                }
2749            }
2750            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2751        }
2752        assert!(content.trim_ascii_end().ends_with(b"\ndone"), "no complete log");
2753        #[cfg(unix)]
2754        {
2755            use std::os::unix::fs::PermissionsExt;
2756            let mode = std::fs::metadata(&logs(&dir)[0]).unwrap().permissions().mode();
2757            assert_eq!(mode & 0o777, 0o600, "request logs are owner-only");
2758        }
2759        drop(app);
2760        drop(state);
2761        // A spool removes its file when dropped, after its section was copied into the
2762        // log, so the log can be complete a moment before the file is gone.
2763        let mut leftovers = Vec::new();
2764        for _ in 0..500 {
2765            leftovers = std::fs::read_dir(&dir)
2766                .unwrap()
2767                .flatten()
2768                .filter(|e| e.path().extension().is_some_and(|e| e == "tmp"))
2769                .collect();
2770            if leftovers.is_empty() {
2771                break;
2772            }
2773            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2774        }
2775        assert!(leftovers.is_empty(), "spool files left behind: {leftovers:?}");
2776        std::fs::remove_dir_all(dir).unwrap();
2777        content
2778    }
2779
2780    fn upstream_request<'a>(
2781        url: &'a str,
2782        headers: &'a [(String, String)],
2783        body: &'a [u8],
2784    ) -> cpa_core::exec::UpstreamRequest<'a> {
2785        cpa_core::exec::UpstreamRequest {
2786            url,
2787            method: "POST",
2788            headers,
2789            body,
2790            provider: "codex",
2791            auth_id: "",
2792            auth_label: "",
2793            auth_type: "",
2794            auth_value: "",
2795        }
2796    }
2797
2798    #[tokio::test]
2799    async fn cookie_values_never_reach_persisted_logs() {
2800        use cpa_core::exec::CaptureEvent::*;
2801        // Request logging on, and off with a forced error log.
2802        for enabled in [true, false] {
2803            let content = persisted(enabled, StatusCode::BAD_GATEWAY, |log| {
2804                let sink = log.capture_sink();
2805                let headers = [
2806                    ("Cookie".into(), "up=c00kie-up".into()),
2807                    ("Content-Type".into(), "application/json".into()),
2808                ];
2809                sink.record(Request(upstream_request("https://fixture.invalid/v1", &headers, b"{}")));
2810                sink.record(ResponseMetadata(
2811                    502,
2812                    &[("Set-Cookie".into(), "up-set=c00kie-up-set".into())],
2813                ));
2814                sink.record(ResponseChunk(b"upstream failed"));
2815            })
2816            .await;
2817            let content = String::from_utf8(content).unwrap();
2818            assert!(!content.contains("c00kie-"), "{content}");
2819            // Downstream request and upstream request.
2820            assert_eq!(content.matches("\nCookie: <redacted>\n").count(), 2, "{content}");
2821            // Downstream response, and the upstream response when it is captured.
2822            assert_eq!(
2823                content.matches("\nSet-Cookie: <redacted>\n").count(),
2824                if enabled { 2 } else { 1 },
2825                "{content}"
2826            );
2827            assert!(content.contains("\nContent-Type: application/json\n"));
2828        }
2829    }
2830
2831    #[tokio::test]
2832    async fn upstream_url_credentials_are_redacted_in_logs() {
2833        use cpa_core::exec::CaptureEvent::*;
2834        let content = persisted(true, StatusCode::OK, |log| {
2835            let sink = log.capture_sink();
2836            sink.record(Request(upstream_request(
2837                "https://alice:pw-h1dden@proxy.example/v1/models?alt=sse&key=AIza-h1dden-query-123&access_token=tok-x-h1dden-x-456#frag",
2838                &[],
2839                b"",
2840            )));
2841            sink.record(WebsocketRequest(upstream_request(
2842                "wss://bob:pw2-h1dden@ws.example/realtime?client_secret=cs-xx-h1dden-xx-789&model=gpt",
2843                &[],
2844                b"",
2845            )));
2846            // A scheme-relative base-url is captured before the transport refuses it.
2847            sink.record(Request(upstream_request("//carol:pw3-h1dden@relay.example/v1", &[], b"")));
2848            sink.record(ResponseError(
2849                "error sending request for uri (https://proxy.example/v1?key=AIza-h1dden-err-1): client error (Connect)",
2850            ));
2851            sink.record(WebsocketError {
2852                stage: "dial",
2853                error: "dial failed: error sending request for uri (ws://127.0.0.1:9/v1?key=QUERY-h1dden-err-2/responses)",
2854            });
2855        })
2856        .await;
2857        let content = String::from_utf8(content).unwrap();
2858        assert!(!content.contains("h1dden"), "{content}");
2859        assert!(
2860            !content.contains("alice") && !content.contains("bob:") && !content.contains("carol"),
2861            "{content}"
2862        );
2863        let line = |prefix: &str| {
2864            content
2865                .lines()
2866                .find(|l| l.starts_with(prefix))
2867                .unwrap_or_else(|| panic!("no {prefix} in {content}"))
2868                .to_owned()
2869        };
2870        let http = line("Upstream URL: https://<redacted>@proxy.example/v1/models?alt=sse&key=");
2871        assert!(http.contains("&access_token=") && http.ends_with("#frag"), "{http}");
2872        let ws = line("Upstream URL: wss://<redacted>@ws.example/realtime?client_secret=");
2873        assert!(ws.ends_with("&model=gpt"), "{ws}");
2874
2875        for url in [
2876            "https://host.example/v1/users/a@b.example?alt=sse",
2877            "http://127.0.0.1:8080/v1#a@b",
2878            "https://host.example/v1?model=gpt",
2879        ] {
2880            assert_eq!(logged_url(url), url);
2881        }
2882        assert_eq!(
2883            logged_url("https://u:p@host.example"),
2884            "https://<redacted>@host.example"
2885        );
2886        assert_eq!(
2887            logged_url("//u:p@relay.example/v1?key=SENTINEL-0006"),
2888            "//<redacted>@relay.example/v1?key=SENT...0006"
2889        );
2890        // Upstream URLs also mask signed and other credential-like names; the client
2891        // query masking keeps Go's names.
2892        let signed = "https://h.example/v1?sig=SIGNATURE-0001&X-Amz-Signature=abcdef-0002&password=hunter-0003&auth=bearer-0004&mykey=keyval-0005&model=gpt";
2893        assert_eq!(
2894            logged_url(signed),
2895            "https://h.example/v1?sig=SIGN...0001&X-Amz-Signature=abcd...0002&password=hunt...0003&auth=bear...0004&mykey=keyv...0005&model=gpt"
2896        );
2897        assert_eq!(
2898            mask_query("sig=SIGNATURE-0001&model=gpt"),
2899            "sig=SIGNATURE-0001&model=gpt"
2900        );
2901        assert_eq!(
2902            logged_url("https://user@host.example/x"),
2903            "https://<redacted>@host.example/x"
2904        );
2905        assert_eq!(
2906            logged_text("uri (https://h.example/v1?key=AIza-0123456789): client error (Connect)"),
2907            "uri (https://h.example/v1?key=AIza...6789): client error (Connect)"
2908        );
2909        assert_eq!(
2910            logged_text("a http://u:p@h.example/x b \"wss://h.example/y?token=abcdefghijkl\" c"),
2911            "a http://<redacted>@h.example/x b \"wss://h.example/y?token=abcd...ijkl\" c"
2912        );
2913        assert_eq!(logged_text("no url: status 500"), "no url: status 500");
2914        // Legal URL punctuation does not end the URL early.
2915        assert_eq!(
2916            logged_text("(https://h.example/a)b?key=SENTINEL-0001)"),
2917            "(https://h.example/a)b?key=SENT...0001)"
2918        );
2919        assert_eq!(
2920            logged_text("https://u:p)w@h.example/x"),
2921            "https://<redacted>@h.example/x"
2922        );
2923        // A URL glued to another is not masked on its own: the span goes whole.
2924        for glued in [
2925            "https://a.example/x,https://u:pw@b.example/y",
2926            "https://a.example/x?alt=sse,https://b.example/y?token=SENTINEL-0002",
2927            "https://a.example/x#f,https://b.example/y?key=SENTINEL-0003",
2928        ] {
2929            assert_eq!(logged_text(&format!("({glued}): x")), "(<redacted>): x", "{glued}");
2930        }
2931        // Multibyte text around a URL.
2932        assert_eq!(
2933            logged_text("éhttps://h.example/v?token=abcdefghijkl é"),
2934            "éhttps://h.example/v?token=abcd...ijkl é"
2935        );
2936        assert_eq!(logged_text("é://x"), "é://x");
2937        // A scheme-like text inside a query value is masked with the value.
2938        assert_eq!(
2939            logged_text("https://h.example/v1?token=abc://SENTINEL-0001"),
2940            "https://h.example/v1?token=abc%3A...0001"
2941        );
2942    }
2943
2944    #[tokio::test]
2945    async fn large_upstream_sections_stay_bounded_in_memory() {
2946        use cpa_core::exec::CaptureEvent::*;
2947        let body = vec![b'q'; 20 << 20];
2948        let chunk: Vec<u8> = (0..64 << 10).map(|i| b'a' + (i % 26) as u8).collect();
2949        const CHUNKS: usize = 640;
2950        let events = {
2951            let body = body.clone();
2952            let chunk = chunk.clone();
2953            move |log: &RequestLog| {
2954                let sink = log.capture_sink();
2955                sink.record(Request(upstream_request("https://fixture.invalid/v1", &[], &body)));
2956                sink.record(ResponseMetadata(200, &[]));
2957                for _ in 0..CHUNKS {
2958                    sink.record(ResponseChunk(&chunk));
2959                }
2960                let locked = log.0.lock().unwrap();
2961                let record = locked.as_ref().unwrap();
2962                for section in [&record.api_request, &record.api_response] {
2963                    assert!(section.spool.is_some());
2964                    assert!(
2965                        section.memory.capacity() <= 2 * SECTION_MEMORY,
2966                        "{}",
2967                        section.memory.capacity()
2968                    );
2969                }
2970            }
2971        };
2972        let content = persisted(true, StatusCode::OK, events).await;
2973        let find = |from: usize, needle: &[u8]| {
2974            from + content[from..]
2975                .windows(needle.len())
2976                .position(|w| w == needle)
2977                .unwrap_or_else(|| panic!("missing {}", String::from_utf8_lossy(needle)))
2978        };
2979        let expect = |at: usize, want: &[u8]| {
2980            assert!(content.len() >= at + want.len() && &content[at..at + want.len()] == want);
2981        };
2982        let request = find(0, b"=== API REQUEST 1 ===\n");
2983        let start = find(request, b"\nBody:\n") + b"\nBody:\n".len();
2984        expect(start, &[&body[..], b"\n\n\n=== API RESPONSE 1 ===\n"].concat());
2985        let response = find(start + body.len(), b"=== API RESPONSE 1 ===\n");
2986        let start = find(response, b"\nBody:\n") + b"\nBody:\n".len();
2987        let mut stream = vec![chunk.clone(); CHUNKS].join(&b"\n\n"[..]);
2988        stream.extend_from_slice(b"\n\n\n=== RESPONSE ===\nStatus: 200\n");
2989        expect(start, &stream);
2990    }
2991}