Skip to main content

cpa_server/
pprof.rs

1//! Optional, separate debug listener (sdk/cliproxy/pprof_server.go, v8.0.35).
2//! One task owns the listener and its connections, so a stale config application or
3//! failed listener cannot stop its replacement. No API or management middleware runs here.
4
5use std::sync::Arc;
6use std::time::Duration;
7
8use axum::Router;
9use axum::extract::{OriginalUri, Request};
10use axum::http::{Method, StatusCode, header};
11use axum::response::{Html, IntoResponse, Response};
12use axum::routing::any;
13use cpa_core::config::Config;
14use hyper_util::rt::{TokioIo, TokioTimer};
15use tokio::net::TcpListener;
16use tokio::sync::watch;
17use tokio::task::{JoinHandle, JoinSet};
18use tower::ServiceExt;
19
20use crate::Runtime;
21
22const STOP_TIMEOUT: Duration = Duration::from_secs(5);
23const PROFILES: &[&str] = &[
24    "allocs",
25    "block",
26    "cmdline",
27    "goroutine",
28    "heap",
29    "mutex",
30    "profile",
31    "symbol",
32    "threadcreate",
33    "trace",
34];
35
36/// The debug listener follows config publications, including Home config updates.
37/// Dropping it requests shutdown; [`Self::shutdown`] waits up to five seconds for
38/// draining. Active requests can finish afterwards, as with Go's http.Server.Shutdown.
39pub struct Server {
40    stop: watch::Sender<bool>,
41    task: Option<JoinHandle<()>>,
42    #[cfg(test)]
43    accept_errors: tokio::sync::mpsc::UnboundedSender<(std::io::Error, tokio::sync::oneshot::Sender<()>)>,
44}
45
46impl Server {
47    pub fn start(rt: &Arc<Runtime>) -> Self {
48        let (stop, stopped) = watch::channel(false);
49        #[cfg(test)]
50        let (accept_errors, mut injected_errors) =
51            tokio::sync::mpsc::unbounded_channel::<(std::io::Error, tokio::sync::oneshot::Sender<()>)>();
52        // Subscribe before taking the first snapshot, so startup cannot miss a reload.
53        let changed = rt.subscribe_config();
54        let weak = Arc::downgrade(rt);
55        let task = tokio::spawn(async move {
56            let mut changed = changed;
57            let mut stopped = stopped;
58            let mut listener: Option<TcpListener> = None;
59            let mut current = (false, String::new());
60            let mut connections = JoinSet::new();
61            let (closing, _) = watch::channel(());
62            let mut apply = true;
63            let mut accept_delay = Duration::ZERO;
64            let mut retry_at = None;
65            loop {
66                if *stopped.borrow() {
67                    break;
68                }
69                if apply {
70                    apply = false;
71                    // Mark this publication before draining. A newer publication
72                    // remains eligible even when this apply times out.
73                    changed.borrow_and_update();
74                    let Some(rt) = weak.upgrade() else { break };
75                    let desired = settings(&rt.config());
76                    drop(rt);
77                    if desired != current || (desired.0 && listener.is_none()) {
78                        // Stop the physical listener before draining; later binds cannot
79                        // accidentally retain connections from the previous generation.
80                        listener = None;
81                        closing.send_replace(());
82                        if tokio::time::timeout(STOP_TIMEOUT, async {
83                            while connections.join_next().await.is_some() {}
84                        })
85                        .await
86                        .is_err()
87                        {
88                            // http.Server.Shutdown's deadline does not abort active requests.
89                            connections.detach_all();
90                            tracing::error!("pprof server stop timed out on {}", current.1);
91                            // Go's ApplyContext returns the shutdown error without binding
92                            // a replacement. Only a later publication can retry.
93                            continue;
94                        }
95                        // A newer configuration may have arrived while old requests drained.
96                        if *stopped.borrow() {
97                            break;
98                        }
99                        changed.borrow_and_update();
100                        let Some(rt) = weak.upgrade() else { break };
101                        current = settings(&rt.config());
102                        drop(rt);
103                        if current.0 {
104                            let binding = async {
105                                if let Some(port) = current.1.strip_prefix(':') {
106                                    let port = port.parse::<u16>().map_err(std::io::Error::other)?;
107                                    TcpListener::from_std(crate::listener::bind("", port)?)
108                                } else {
109                                    TcpListener::bind(&current.1).await
110                                }
111                            };
112                            // Hostname resolution must not hold shutdown or a newer config hostage.
113                            let bound = tokio::select! {
114                                biased;
115                                _ = stopped.changed() => break,
116                                result = changed.changed() => {
117                                    if result.is_err() { break; }
118                                    apply = true;
119                                    continue;
120                                },
121                                bound = binding => bound,
122                            };
123                            match bound {
124                                Ok(bound) => {
125                                    tracing::info!("pprof server starting on {}", current.1);
126                                    listener = Some(bound);
127                                    accept_delay = Duration::ZERO;
128                                    retry_at = None;
129                                }
130                                Err(error) => tracing::error!("pprof server failed on {}: {error}", current.1),
131                            }
132                        }
133                    }
134                }
135                if *stopped.borrow() {
136                    break;
137                }
138                tokio::select! {
139                    biased;
140                    _ = stopped.changed() => break,
141                    result = changed.changed() => {
142                        if result.is_err() { break; }
143                        apply = true;
144                    },
145                    _ = connections.join_next(), if !connections.is_empty() => {},
146                    accepted = async {
147                        let Some(listener) = &listener else {
148                            return std::future::pending().await;
149                        };
150                        if let Some(deadline) = retry_at {
151                            tokio::time::sleep_until(deadline).await;
152                        }
153                        #[cfg(test)]
154                        {
155                            tokio::select! {
156                                biased;
157                                error = injected_errors.recv() => {
158                                    let (error, accepted) = error.expect("server owns injection sender");
159                                    let _ = accepted.send(());
160                                    Err(error)
161                                },
162                                accepted = listener.accept() => accepted,
163                            }
164                        }
165                        #[cfg(not(test))]
166                        listener.accept().await
167                    } => match accepted {
168                        Ok((stream, _)) => {
169                            accept_delay = Duration::ZERO;
170                            retry_at = None;
171                            let mut closing = closing.subscribe();
172                            connections.spawn(async move {
173                                let app = router();
174                                let service = hyper::service::service_fn(move |req| app.clone().oneshot(req));
175                                let mut builder = hyper::server::conn::http1::Builder::new();
176                                builder.timer(TokioTimer::new()).header_read_timeout(STOP_TIMEOUT);
177                                let connection = builder.serve_connection(TokioIo::new(stream), service);
178                                tokio::pin!(connection);
179                                tokio::select! {
180                                    _ = &mut connection => {},
181                                    _ = closing.changed() => {
182                                        connection.as_mut().graceful_shutdown();
183                                        let _ = connection.await;
184                                    }
185                                }
186                            });
187                        }
188                        Err(error) if temporary_accept_error(&error) => {
189                            // net/http.Server.Serve keeps its socket and active
190                            // connections, retrying from 5 ms up to one second.
191                            accept_delay = if accept_delay.is_zero() {
192                                Duration::from_millis(5)
193                            } else {
194                                (accept_delay * 2).min(Duration::from_secs(1))
195                            };
196                            retry_at = Some(tokio::time::Instant::now() + accept_delay);
197                            tracing::error!("pprof accept failed on {}: {error}; retrying in {accept_delay:?}", current.1);
198                        }
199                        Err(error) => {
200                            tracing::error!("pprof accept failed on {}: {error}", current.1);
201                            // A terminal failure waits for another config publication.
202                            listener = None;
203                        }
204                    }
205                }
206            }
207            drop(listener);
208            closing.send_replace(());
209            if tokio::time::timeout(STOP_TIMEOUT, async { while connections.join_next().await.is_some() {} })
210                .await
211                .is_err()
212            {
213                connections.detach_all();
214                tracing::error!("pprof server stop timed out on {}", current.1);
215            }
216        });
217        Self {
218            stop,
219            task: Some(task),
220            #[cfg(test)]
221            accept_errors,
222        }
223    }
224
225    pub async fn shutdown(mut self) {
226        self.stop.send_replace(true);
227        if let Some(task) = self.task.take() {
228            let _ = task.await;
229        }
230    }
231}
232
233impl Drop for Server {
234    fn drop(&mut self) {
235        self.stop.send_replace(true);
236    }
237}
238
239fn temporary_accept_error(error: &std::io::Error) -> bool {
240    use std::io::ErrorKind::{ConnectionAborted, ConnectionReset, Interrupted, TimedOut, WouldBlock};
241    if matches!(
242        error.kind(),
243        ConnectionAborted | ConnectionReset | Interrupted | TimedOut | WouldBlock
244    ) {
245        return true;
246    }
247    // net.OpError.Temporary also includes exhausted process/system descriptor tables.
248    #[cfg(unix)]
249    if matches!(error.raw_os_error(), Some(libc::EMFILE | libc::ENFILE)) {
250        return true;
251    }
252    #[cfg(windows)]
253    if error.raw_os_error() == Some(10024) {
254        return true;
255    }
256    false
257}
258
259fn settings(config: &Config) -> (bool, String) {
260    let pprof = config.document.get("observability").and_then(|o| o.get("pprof"));
261    let enabled = pprof.and_then(|p| p.get("enable")).and_then(|v| v.as_bool()) == Some(true);
262    let addr = pprof.and_then(|p| p.get("addr")).and_then(|v| v.as_str()).unwrap_or("");
263    let addr = crate::gojson::trim(addr);
264    (enabled, if addr.is_empty() { "127.0.0.1:8316" } else { addr }.into())
265}
266
267fn router() -> Router {
268    Router::new()
269        .route(
270            "/debug/pprof",
271            any(|OriginalUri(uri): OriginalUri| async move {
272                let location = match uri.query() {
273                    Some(query) => format!("/debug/pprof/?{query}"),
274                    None => "/debug/pprof/".into(),
275                };
276                // Go 1.26 ServeMux (CLIProxyAPI v8.0.35's release toolchain) answers 307.
277                (StatusCode::TEMPORARY_REDIRECT, [(header::LOCATION, location)])
278            }),
279        )
280        .route("/debug/pprof/", any(index))
281        .route("/debug/pprof/cmdline", any(cmdline))
282        .route("/debug/pprof/symbol", any(symbol))
283        .route("/debug/pprof/profile", any(profile))
284        .route("/debug/pprof/{name}", any(unsupported))
285}
286
287fn text(status: StatusCode, message: impl Into<String>) -> Response {
288    (status, [(header::X_CONTENT_TYPE_OPTIONS, "nosniff")], message.into()).into_response()
289}
290
291async fn index() -> Response {
292    let mut html = String::from("<html><head><title>/debug/pprof/</title></head><body><h1>/debug/pprof/</h1><ul>");
293    for name in PROFILES {
294        html.push_str(&format!("<li><a href=\"{name}\">{name}</a></li>\n"));
295    }
296    html.push_str("</ul><p>Go runtime profiles are unavailable in Rust. CPU profiling requires the pprof build feature.</p></body></html>");
297    Html(html).into_response()
298}
299
300async fn cmdline() -> Response {
301    // net/http/pprof.Cmdline joins os.Args with NUL, with no trailing separator.
302    text(
303        StatusCode::OK,
304        std::env::args_os()
305            .map(|s| s.to_string_lossy().into_owned())
306            .collect::<Vec<_>>()
307            .join("\0"),
308    )
309}
310
311async fn symbol(OriginalUri(uri): OriginalUri, req: Request) -> Response {
312    let raw = if req.method() == Method::POST {
313        match axum::body::to_bytes(req.into_body(), crate::MAX_REQUEST_BYTES).await {
314            Ok(body) => String::from_utf8_lossy(&body).into_owned(),
315            Err(_) => return text(StatusCode::PAYLOAD_TOO_LARGE, "symbol body too large\n"),
316        }
317    } else {
318        uri.query().unwrap_or_default().to_owned()
319    };
320    #[cfg(not(feature = "pprof"))]
321    {
322        let _ = raw;
323        text(StatusCode::OK, "num_symbols: 1\n")
324    }
325    #[cfg(feature = "pprof")]
326    {
327        // Native symbolization takes a blocking global lock. Bound dispatched jobs, not
328        // just awaiting requests: a cancelled request keeps its slot until the job stops.
329        static SYMBOLS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
330        let capacity = SYMBOLS.acquire().await.expect("symbol semaphore is never closed");
331        let (active, cancelled) = watch::channel(());
332        let result = tokio::task::spawn_blocking(move || {
333            let _capacity = capacity;
334            let mut output = String::from("num_symbols: 1\n");
335            for pc in raw.split('+') {
336                if cancelled.has_changed().is_err() {
337                    break;
338                }
339                let parsed = if let Some(hex) = pc.strip_prefix("0x").or_else(|| pc.strip_prefix("0X")) {
340                    usize::from_str_radix(hex, 16)
341                } else if pc.starts_with('0') {
342                    usize::from_str_radix(pc, 8)
343                } else {
344                    pc.parse()
345                };
346                if let Ok(addr) = parsed
347                    && addr != 0
348                {
349                    // backtrace subtracts one from a return address; pprof takes the actual PC.
350                    backtrace::resolve(addr.wrapping_add(1) as *mut std::ffi::c_void, |symbol| {
351                        if let Some(name) = symbol.name() {
352                            output.push_str(&format!("{addr:#x} {name}\n"));
353                        }
354                    });
355                }
356            }
357            output
358        })
359        .await;
360        drop(active);
361        match result {
362            Ok(output) => text(StatusCode::OK, output),
363            Err(_) => text(StatusCode::INTERNAL_SERVER_ERROR, "Could not resolve symbols\n"),
364        }
365    }
366}
367
368async fn unsupported(axum::extract::Path(name): axum::extract::Path<String>) -> Response {
369    if PROFILES.contains(&name.as_str()) {
370        text(
371            StatusCode::NOT_IMPLEMENTED,
372            format!("{name}: Go runtime profiles are unavailable in Rust\n"),
373        )
374    } else {
375        text(StatusCode::NOT_FOUND, "Unknown profile\n")
376    }
377}
378
379#[cfg(not(all(feature = "pprof", unix)))]
380async fn profile() -> Response {
381    text(
382        StatusCode::NOT_IMPLEMENTED,
383        "CPU profiling requires the cpa-server/pprof build feature on Unix\n",
384    )
385}
386
387#[cfg(all(feature = "pprof", unix))]
388async fn profile(OriginalUri(uri): OriginalUri, req: Request) -> Response {
389    use pprof::protos::Message;
390    // net/http FormValue prefers POST, PUT and PATCH form bodies to the URL query.
391    let mut seconds = None;
392    if matches!(*req.method(), Method::POST | Method::PUT | Method::PATCH)
393        // mime.ParseMediaType: the type is case-insensitive, parameters and spaces ignored.
394        && req.headers().get(header::CONTENT_TYPE).is_some_and(|v| {
395            v.to_str().unwrap_or_default().split(';').next().unwrap_or_default().trim()
396                .eq_ignore_ascii_case("application/x-www-form-urlencoded")
397        })
398    {
399        let Ok(body) = axum::body::to_bytes(req.into_body(), crate::MAX_REQUEST_BYTES).await else {
400            return text(StatusCode::PAYLOAD_TOO_LARGE, "profile body too large\n");
401        };
402        seconds = crate::access::query_get(&String::from_utf8_lossy(&body), "seconds");
403    }
404    // Go defaults invalid, zero and negative durations to 30 seconds.
405    let seconds = seconds
406        .or_else(|| crate::access::query_get(uri.query().unwrap_or_default(), "seconds"))
407        .and_then(|s| String::from_utf8(s).ok())
408        .and_then(|s| s.parse::<i64>().ok())
409        .filter(|n| *n > 0)
410        .unwrap_or(30);
411    // Go time.Duration is signed nanoseconds and multiplication wraps.
412    let duration = Duration::from_nanos(seconds.wrapping_mul(1_000_000_000).max(0) as u64);
413    // Reject overlap before touching native locks, including initialization/reporting.
414    static CPU: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1);
415    let Ok(permit) = CPU.try_acquire() else {
416        return text(StatusCode::INTERNAL_SERVER_ERROR, "Could not enable CPU profiling\n");
417    };
418    let started = tokio::task::spawn_blocking(move || {
419        pprof::ProfilerGuardBuilder::default()
420            .frequency(100)
421            .blocklist(&["libc", "libgcc", "pthread", "vdso"])
422            .build()
423            .map(|guard| (guard, permit))
424    })
425    .await;
426    let (guard, permit) = match started {
427        Ok(Ok(started)) => started,
428        _ => return text(StatusCode::INTERNAL_SERVER_ERROR, "Could not enable CPU profiling\n"),
429    };
430    // Locals drop in reverse declaration order: stop sampling before releasing admission.
431    let guard = guard;
432    // Keep the guard in the request future: cancelling it releases the global profiler.
433    tokio::time::sleep(duration).await;
434    // Native unwinding/symbolization is blocking. Only the one global profiler can
435    // reach this worker. Cancellation during sampling still drops the guard above.
436    let bytes = tokio::task::spawn_blocking(move || {
437        let _permit = permit;
438        let report = guard.report().build();
439        drop(guard);
440        report
441            .and_then(|r| r.pprof())
442            .ok()
443            .and_then(|p| p.write_to_bytes().ok())
444    })
445    .await
446    .ok()
447    .flatten();
448    match bytes {
449        Some(bytes) => (
450            [
451                (header::CONTENT_TYPE, "application/octet-stream"),
452                (header::CONTENT_DISPOSITION, "attachment; filename=\"profile\""),
453                (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
454            ],
455            bytes,
456        )
457            .into_response(),
458        None => text(StatusCode::INTERNAL_SERVER_ERROR, "Could not build CPU profile\n"),
459    }
460}
461
462#[cfg(test)]
463mod tests {
464    use super::*;
465    use tokio::io::{AsyncReadExt, AsyncWriteExt};
466
467    async fn symbol_on(stream: &mut tokio::net::TcpStream) {
468        stream
469            .write_all(b"GET /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\n\r\n")
470            .await
471            .unwrap();
472        let mut headers = Vec::new();
473        while !headers.ends_with(b"\r\n\r\n") {
474            headers.push(stream.read_u8().await.unwrap());
475        }
476        assert!(headers.starts_with(b"HTTP/1.1 200"));
477        let mut body = [0; 15];
478        stream.read_exact(&mut body).await.unwrap();
479        assert_eq!(&body, b"num_symbols: 1\n");
480    }
481
482    #[tokio::test]
483    async fn pprof_routes_and_symbols() {
484        let response = router()
485            .oneshot(
486                Request::builder()
487                    .method(Method::POST)
488                    .uri("/debug/pprof?seconds=2")
489                    .body(axum::body::Body::empty())
490                    .unwrap(),
491            )
492            .await
493            .unwrap();
494        assert_eq!(response.status(), 307);
495        assert_eq!(response.headers()[header::LOCATION], "/debug/pprof/?seconds=2");
496        for (path, expected) in [("", 200), ("cmdline", 200), ("symbol", 200), ("unknown", 404)] {
497            let response = router()
498                .oneshot(
499                    Request::builder()
500                        .uri(format!("/debug/pprof/{path}"))
501                        .body(axum::body::Body::empty())
502                        .unwrap(),
503                )
504                .await
505                .unwrap();
506            assert_eq!(response.status(), expected, "{path}");
507        }
508        for name in ["allocs", "block", "goroutine", "heap", "mutex", "threadcreate", "trace"] {
509            let response = router()
510                .oneshot(
511                    Request::builder()
512                        .uri(format!("/debug/pprof/{name}"))
513                        .body(axum::body::Body::empty())
514                        .unwrap(),
515                )
516                .await
517                .unwrap();
518            assert_eq!(response.status(), 501, "{name}");
519            assert!(
520                response.headers()[header::CONTENT_TYPE]
521                    .to_str()
522                    .unwrap()
523                    .starts_with("text/plain")
524            );
525        }
526        let addr = pprof_routes_and_symbols as *const () as usize;
527        for method in [Method::GET, Method::POST] {
528            let raw = format!("{addr:#x}+0x0+garbage");
529            let uri = if method == Method::GET {
530                format!("/debug/pprof/symbol?{raw}")
531            } else {
532                "/debug/pprof/symbol".into()
533            };
534            let response = router()
535                .oneshot(
536                    Request::builder()
537                        .method(method)
538                        .uri(uri)
539                        .body(axum::body::Body::from(raw))
540                        .unwrap(),
541                )
542                .await
543                .unwrap();
544            let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
545            let body = String::from_utf8(body.to_vec()).unwrap();
546            assert!(body.starts_with("num_symbols: 1\n"));
547            #[cfg(feature = "pprof")]
548            assert!(body.contains("pprof_routes_and_symbols"), "{body}");
549            #[cfg(not(feature = "pprof"))]
550            assert_eq!(body, "num_symbols: 1\n");
551        }
552        #[cfg(not(feature = "pprof"))]
553        {
554            let response = router()
555                .oneshot(
556                    Request::builder()
557                        .uri("/debug/pprof/profile")
558                        .body(axum::body::Body::empty())
559                        .unwrap(),
560                )
561                .await
562                .unwrap();
563            assert_eq!(response.status(), 501);
564            let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
565            assert!(String::from_utf8_lossy(&body).contains("cpa-server/pprof"));
566        }
567    }
568
569    #[tokio::test]
570    async fn pprof_listener_reload_failure_and_shutdown() {
571        let dir = std::env::temp_dir().join(format!("pprof-{}", uuid::Uuid::new_v4()));
572        std::fs::create_dir_all(&dir).unwrap();
573        let config = |enabled, addr: &str, legacy| {
574            let pprof = format!("pprof: {{enable: {enabled}, addr: '{addr}'}}");
575            Config::parse(&format!(
576                "oauth: {{auth-dir: '{}'}}\n{}\n",
577                dir.display(),
578                if legacy {
579                    pprof
580                } else {
581                    format!("observability:\n  {pprof}")
582                }
583            ))
584            .unwrap()
585        };
586        let executors = cpa_exec::Executors {
587            claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
588            codex: Default::default(),
589            devices: Default::default(),
590            openai: Default::default(),
591            google: Default::default(),
592        };
593        let occupied = TcpListener::bind("127.0.0.1:0").await.unwrap();
594        let addr = occupied.local_addr().unwrap().to_string();
595        let rt = Arc::new(crate::testing::runtime(config(false, &addr, false), vec![], executors));
596        let server = Server::start(&rt);
597        tokio::task::yield_now().await;
598        rt.publish_config(config(true, &addr, false));
599        tokio::task::yield_now().await;
600        // A failed bind is retried on a later same-address publication.
601        drop(occupied);
602        rt.publish_config(config(true, &addr, true));
603        let client = wreq::Client::builder().no_proxy().build().unwrap();
604        let fetch = |addr: String| {
605            let client = client.clone();
606            async move {
607                tokio::time::timeout(Duration::from_secs(7), async {
608                    loop {
609                        if let Ok(response) = client.get(format!("http://{addr}/debug/pprof/cmdline")).send().await {
610                            assert_eq!(response.status(), 200);
611                            let expected = std::env::args_os()
612                                .map(|s| s.to_string_lossy().into_owned())
613                                .collect::<Vec<_>>()
614                                .join("\0");
615                            assert_eq!(response.text().await.unwrap(), expected);
616                            break;
617                        }
618                        tokio::time::sleep(Duration::from_millis(10)).await;
619                    }
620                })
621                .await
622                .unwrap();
623            }
624        };
625        fetch(addr.clone()).await;
626        // An unrelated/same-address reload does not close an existing connection.
627        let mut keep = tokio::net::TcpStream::connect(&addr).await.unwrap();
628        symbol_on(&mut keep).await;
629        rt.publish_config(config(true, &addr, false));
630        tokio::task::yield_now().await;
631        symbol_on(&mut keep).await;
632        // Inject repeated descriptor exhaustion at the real accept boundary.
633        // Retries back off, but neither the socket nor existing connections close.
634        let started = tokio::time::Instant::now();
635        for _ in 0..9 {
636            let (accepted, observed) = tokio::sync::oneshot::channel();
637            #[cfg(unix)]
638            let error = std::io::Error::from_raw_os_error(libc::EMFILE);
639            #[cfg(windows)]
640            let error = std::io::Error::from_raw_os_error(10024);
641            server.accept_errors.send((error, accepted)).unwrap();
642            observed.await.unwrap();
643            symbol_on(&mut keep).await;
644        }
645        assert!(
646            started.elapsed() >= Duration::from_secs(1),
647            "accept errors did not back off"
648        );
649        // The current retry delay is one second; a disable must interrupt it.
650        rt.publish_config(config(false, &addr, false));
651        tokio::time::timeout(Duration::from_millis(500), async {
652            while tokio::net::TcpStream::connect(&addr).await.is_ok() {
653                tokio::task::yield_now().await;
654            }
655        })
656        .await
657        .unwrap();
658        rt.publish_config(config(true, &addr, false));
659        fetch(addr.clone()).await;
660        // Hold a request body open past the drain deadline. Go leaves it running
661        // and does not bind the replacement until a later config publication.
662        let mut active = tokio::net::TcpStream::connect(&addr).await.unwrap();
663        active.write_all(b"POST /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\n").await.unwrap();
664        let mut interim = [0; 25];
665        active.read_exact(&mut interim).await.unwrap();
666        assert_eq!(&interim, b"HTTP/1.1 100 Continue\r\n\r\n");
667        let replacement = TcpListener::bind("127.0.0.1:0").await.unwrap();
668        let stale = replacement.local_addr().unwrap().to_string();
669        drop(replacement);
670        rt.publish_config(config(true, &stale, false));
671        tokio::time::timeout(Duration::from_secs(5), async {
672            while tokio::net::TcpStream::connect(&addr).await.is_ok() {
673                tokio::time::sleep(Duration::from_millis(10)).await;
674            }
675        })
676        .await
677        .unwrap();
678        tokio::time::sleep(STOP_TIMEOUT + Duration::from_millis(100)).await;
679        assert!(tokio::net::TcpStream::connect(&stale).await.is_err());
680        let replacement = crate::listener::bind("", 0).unwrap();
681        let port = replacement.local_addr().unwrap().port();
682        let ipv6 = replacement.local_addr().unwrap().is_ipv6();
683        let next = format!("127.0.0.1:{port}");
684        let wildcard = format!(":{port}");
685        drop(replacement);
686        rt.publish_config(config(true, &wildcard, false));
687        // The next publication remains eligible while the detached request runs.
688        fetch(next.clone()).await;
689        active.write_all(b"0x0").await.unwrap();
690        let mut completed = String::new();
691        tokio::time::timeout(Duration::from_secs(5), active.read_to_string(&mut completed))
692            .await
693            .unwrap()
694            .unwrap();
695        assert!(completed.starts_with("HTTP/1.1 200"), "{completed}");
696        assert!(completed.contains("num_symbols: 1"), "{completed}");
697        assert!(tokio::net::TcpStream::connect(&addr).await.is_err());
698        assert!(tokio::net::TcpStream::connect(&stale).await.is_err());
699        if ipv6 {
700            fetch(format!("[::1]:{port}")).await;
701        }
702        rt.publish_config(config(false, &next, false));
703        tokio::time::timeout(Duration::from_secs(5), async {
704            while tokio::net::TcpStream::connect(&next).await.is_ok() {
705                tokio::time::sleep(Duration::from_millis(10)).await;
706            }
707        })
708        .await
709        .unwrap();
710        rt.publish_config(config(true, &next, false));
711        fetch(next.clone()).await;
712        // Terminal accept failures do not spin on rebind or close active sockets.
713        let mut keep = tokio::net::TcpStream::connect(&next).await.unwrap();
714        symbol_on(&mut keep).await;
715        let mut held = tokio::net::TcpStream::connect(&next).await.unwrap();
716        held.write_all(b"POST /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\n").await.unwrap();
717        let mut interim = [0; 25];
718        tokio::time::timeout(Duration::from_secs(5), held.read_exact(&mut interim))
719            .await
720            .unwrap()
721            .unwrap();
722        assert_eq!(&interim, b"HTTP/1.1 100 Continue\r\n\r\n");
723        let (accepted, observed) = tokio::sync::oneshot::channel();
724        server
725            .accept_errors
726            .send((std::io::Error::other("terminal accept failure"), accepted))
727            .unwrap();
728        observed.await.unwrap();
729        symbol_on(&mut keep).await;
730        tokio::time::sleep(Duration::from_millis(50)).await;
731        assert!(tokio::net::TcpStream::connect(&next).await.is_err());
732        // Unlike Go, the next publication drains surviving connections before binding:
733        // the held request keeps running and the replacement waits for it.
734        rt.publish_config(config(true, &next, false));
735        // EOF on the idle connection proves the publication started its graceful drain.
736        let mut rest = Vec::new();
737        let read = tokio::time::timeout(Duration::from_secs(3), keep.read_to_end(&mut rest)).await;
738        assert!(matches!(read, Ok(Ok(0))), "{read:?}");
739        // Draining waits for the held request, so nothing is bound yet.
740        assert!(tokio::net::TcpStream::connect(&next).await.is_err());
741        // Released well inside the five-second drain. Had the drain timed out instead,
742        // the request would be detached and nothing rebound, so `fetch` below fails
743        // within its own seven-second bound.
744        held.write_all(b"0x0").await.unwrap();
745        let mut completed = String::new();
746        tokio::time::timeout(Duration::from_secs(5), held.read_to_string(&mut completed))
747            .await
748            .unwrap()
749            .unwrap();
750        assert!(completed.starts_with("HTTP/1.1 200"), "{completed}");
751        fetch(next.clone()).await;
752        server.shutdown().await;
753        assert!(tokio::net::TcpStream::connect(&next).await.is_err());
754        std::fs::remove_dir_all(dir).unwrap();
755    }
756
757    #[cfg(all(feature = "pprof", unix))]
758    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
759    async fn pprof_cpu_profile_is_decodable_and_exclusive() {
760        use pprof::protos::Message;
761        let request = || {
762            Request::builder()
763                .uri("/debug/pprof/profile?seconds=1")
764                .body(axum::body::Body::empty())
765                .unwrap()
766        };
767        let mut first = Box::pin(router().oneshot(request()));
768        assert!(
769            tokio::time::timeout(Duration::from_millis(30), &mut first)
770                .await
771                .is_err()
772        );
773        let first = tokio::spawn(first);
774        // Keep real CPU work running throughout sampling, not just during native
775        // initialization (which can exceed 200 ms on a loaded CI worker).
776        let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
777        let stopped = stop.clone();
778        let workload = std::thread::spawn(move || {
779            while !stopped.load(std::sync::atomic::Ordering::Relaxed) {
780                std::hint::black_box((0u64..1000).fold(0u64, |sum, n| sum.wrapping_add(n)));
781            }
782        });
783        let second = router().oneshot(request()).await.unwrap();
784        assert_eq!(second.status(), 500);
785        let first = first.await.unwrap().unwrap();
786        stop.store(true, std::sync::atomic::Ordering::Relaxed);
787        workload.join().unwrap();
788        assert_eq!(first.status(), 200);
789        assert_eq!(first.headers()[header::CONTENT_TYPE], "application/octet-stream");
790        let bytes = axum::body::to_bytes(first.into_body(), usize::MAX).await.unwrap();
791        let decoded = pprof::protos::Profile::parse_from_bytes(&bytes).unwrap();
792        assert!(decoded.duration_nanos > 0);
793        assert!(!decoded.sample.is_empty(), "profile did not sample the CPU workload");
794        // Cancellation frees the global profiler; a later request can acquire it.
795        let mut pending = Box::pin(router().oneshot(request()));
796        assert!(
797            tokio::time::timeout(Duration::from_millis(30), &mut pending)
798                .await
799                .is_err()
800        );
801        drop(pending);
802        let types = [
803            "application/x-www-form-urlencoded",
804            " Application/X-Www-Form-Urlencoded ; charset=utf-8",
805        ];
806        for (method, content_type) in [Method::POST, Method::PUT, Method::PATCH]
807            .into_iter()
808            .flat_map(|m| types.map(|t| (m.clone(), t)))
809        {
810            let response = tokio::time::timeout(
811                Duration::from_secs(5),
812                router().oneshot(
813                    Request::builder()
814                        .method(method)
815                        .uri("/debug/pprof/profile?seconds=30")
816                        .header(header::CONTENT_TYPE, content_type)
817                        .body(axum::body::Body::from("seconds=1"))
818                        .unwrap(),
819                ),
820            )
821            .await
822            .unwrap()
823            .unwrap();
824            assert_eq!(response.status(), 200);
825        }
826    }
827}