1use std::sync::Arc;
6use std::time::Duration;
7
8use axum::Router;
9use axum::extract::{OriginalUri, Request};
10use axum::http::{Method, StatusCode, header};
11use axum::response::{Html, IntoResponse, Response};
12use axum::routing::any;
13use cpa_core::config::Config;
14use hyper_util::rt::{TokioIo, TokioTimer};
15use tokio::net::TcpListener;
16use tokio::sync::watch;
17use tokio::task::{JoinHandle, JoinSet};
18use tower::ServiceExt;
19
20use crate::Runtime;
21
22const STOP_TIMEOUT: Duration = Duration::from_secs(5);
23const PROFILES: &[&str] = &[
24 "allocs",
25 "block",
26 "cmdline",
27 "goroutine",
28 "heap",
29 "mutex",
30 "profile",
31 "symbol",
32 "threadcreate",
33 "trace",
34];
35
36pub struct Server {
40 stop: watch::Sender<bool>,
41 task: Option<JoinHandle<()>>,
42 #[cfg(test)]
43 accept_errors: tokio::sync::mpsc::UnboundedSender<(std::io::Error, tokio::sync::oneshot::Sender<()>)>,
44}
45
46impl Server {
47 pub fn start(rt: &Arc<Runtime>) -> Self {
48 let (stop, stopped) = watch::channel(false);
49 #[cfg(test)]
50 let (accept_errors, mut injected_errors) =
51 tokio::sync::mpsc::unbounded_channel::<(std::io::Error, tokio::sync::oneshot::Sender<()>)>();
52 let changed = rt.subscribe_config();
54 let weak = Arc::downgrade(rt);
55 let task = tokio::spawn(async move {
56 let mut changed = changed;
57 let mut stopped = stopped;
58 let mut listener: Option<TcpListener> = None;
59 let mut current = (false, String::new());
60 let mut connections = JoinSet::new();
61 let (closing, _) = watch::channel(());
62 let mut apply = true;
63 let mut accept_delay = Duration::ZERO;
64 let mut retry_at = None;
65 loop {
66 if *stopped.borrow() {
67 break;
68 }
69 if apply {
70 apply = false;
71 changed.borrow_and_update();
74 let Some(rt) = weak.upgrade() else { break };
75 let desired = settings(&rt.config());
76 drop(rt);
77 if desired != current || (desired.0 && listener.is_none()) {
78 listener = None;
81 closing.send_replace(());
82 if tokio::time::timeout(STOP_TIMEOUT, async {
83 while connections.join_next().await.is_some() {}
84 })
85 .await
86 .is_err()
87 {
88 connections.detach_all();
90 tracing::error!("pprof server stop timed out on {}", current.1);
91 continue;
94 }
95 if *stopped.borrow() {
97 break;
98 }
99 changed.borrow_and_update();
100 let Some(rt) = weak.upgrade() else { break };
101 current = settings(&rt.config());
102 drop(rt);
103 if current.0 {
104 let binding = async {
105 if let Some(port) = current.1.strip_prefix(':') {
106 let port = port.parse::<u16>().map_err(std::io::Error::other)?;
107 TcpListener::from_std(crate::listener::bind("", port)?)
108 } else {
109 TcpListener::bind(¤t.1).await
110 }
111 };
112 let bound = tokio::select! {
114 biased;
115 _ = stopped.changed() => break,
116 result = changed.changed() => {
117 if result.is_err() { break; }
118 apply = true;
119 continue;
120 },
121 bound = binding => bound,
122 };
123 match bound {
124 Ok(bound) => {
125 tracing::info!("pprof server starting on {}", current.1);
126 listener = Some(bound);
127 accept_delay = Duration::ZERO;
128 retry_at = None;
129 }
130 Err(error) => tracing::error!("pprof server failed on {}: {error}", current.1),
131 }
132 }
133 }
134 }
135 if *stopped.borrow() {
136 break;
137 }
138 tokio::select! {
139 biased;
140 _ = stopped.changed() => break,
141 result = changed.changed() => {
142 if result.is_err() { break; }
143 apply = true;
144 },
145 _ = connections.join_next(), if !connections.is_empty() => {},
146 accepted = async {
147 let Some(listener) = &listener else {
148 return std::future::pending().await;
149 };
150 if let Some(deadline) = retry_at {
151 tokio::time::sleep_until(deadline).await;
152 }
153 #[cfg(test)]
154 {
155 tokio::select! {
156 biased;
157 error = injected_errors.recv() => {
158 let (error, accepted) = error.expect("server owns injection sender");
159 let _ = accepted.send(());
160 Err(error)
161 },
162 accepted = listener.accept() => accepted,
163 }
164 }
165 #[cfg(not(test))]
166 listener.accept().await
167 } => match accepted {
168 Ok((stream, _)) => {
169 accept_delay = Duration::ZERO;
170 retry_at = None;
171 let mut closing = closing.subscribe();
172 connections.spawn(async move {
173 let app = router();
174 let service = hyper::service::service_fn(move |req| app.clone().oneshot(req));
175 let mut builder = hyper::server::conn::http1::Builder::new();
176 builder.timer(TokioTimer::new()).header_read_timeout(STOP_TIMEOUT);
177 let connection = builder.serve_connection(TokioIo::new(stream), service);
178 tokio::pin!(connection);
179 tokio::select! {
180 _ = &mut connection => {},
181 _ = closing.changed() => {
182 connection.as_mut().graceful_shutdown();
183 let _ = connection.await;
184 }
185 }
186 });
187 }
188 Err(error) if temporary_accept_error(&error) => {
189 accept_delay = if accept_delay.is_zero() {
192 Duration::from_millis(5)
193 } else {
194 (accept_delay * 2).min(Duration::from_secs(1))
195 };
196 retry_at = Some(tokio::time::Instant::now() + accept_delay);
197 tracing::error!("pprof accept failed on {}: {error}; retrying in {accept_delay:?}", current.1);
198 }
199 Err(error) => {
200 tracing::error!("pprof accept failed on {}: {error}", current.1);
201 listener = None;
203 }
204 }
205 }
206 }
207 drop(listener);
208 closing.send_replace(());
209 if tokio::time::timeout(STOP_TIMEOUT, async { while connections.join_next().await.is_some() {} })
210 .await
211 .is_err()
212 {
213 connections.detach_all();
214 tracing::error!("pprof server stop timed out on {}", current.1);
215 }
216 });
217 Self {
218 stop,
219 task: Some(task),
220 #[cfg(test)]
221 accept_errors,
222 }
223 }
224
225 pub async fn shutdown(mut self) {
226 self.stop.send_replace(true);
227 if let Some(task) = self.task.take() {
228 let _ = task.await;
229 }
230 }
231}
232
233impl Drop for Server {
234 fn drop(&mut self) {
235 self.stop.send_replace(true);
236 }
237}
238
239fn temporary_accept_error(error: &std::io::Error) -> bool {
240 use std::io::ErrorKind::{ConnectionAborted, ConnectionReset, Interrupted, TimedOut, WouldBlock};
241 if matches!(
242 error.kind(),
243 ConnectionAborted | ConnectionReset | Interrupted | TimedOut | WouldBlock
244 ) {
245 return true;
246 }
247 #[cfg(unix)]
249 if matches!(error.raw_os_error(), Some(libc::EMFILE | libc::ENFILE)) {
250 return true;
251 }
252 #[cfg(windows)]
253 if error.raw_os_error() == Some(10024) {
254 return true;
255 }
256 false
257}
258
259fn settings(config: &Config) -> (bool, String) {
260 let pprof = config.document.get("observability").and_then(|o| o.get("pprof"));
261 let enabled = pprof.and_then(|p| p.get("enable")).and_then(|v| v.as_bool()) == Some(true);
262 let addr = pprof.and_then(|p| p.get("addr")).and_then(|v| v.as_str()).unwrap_or("");
263 let addr = crate::gojson::trim(addr);
264 (enabled, if addr.is_empty() { "127.0.0.1:8316" } else { addr }.into())
265}
266
267fn router() -> Router {
268 Router::new()
269 .route(
270 "/debug/pprof",
271 any(|OriginalUri(uri): OriginalUri| async move {
272 let location = match uri.query() {
273 Some(query) => format!("/debug/pprof/?{query}"),
274 None => "/debug/pprof/".into(),
275 };
276 (StatusCode::TEMPORARY_REDIRECT, [(header::LOCATION, location)])
278 }),
279 )
280 .route("/debug/pprof/", any(index))
281 .route("/debug/pprof/cmdline", any(cmdline))
282 .route("/debug/pprof/symbol", any(symbol))
283 .route("/debug/pprof/profile", any(profile))
284 .route("/debug/pprof/{name}", any(unsupported))
285}
286
287fn text(status: StatusCode, message: impl Into<String>) -> Response {
288 (status, [(header::X_CONTENT_TYPE_OPTIONS, "nosniff")], message.into()).into_response()
289}
290
291async fn index() -> Response {
292 let mut html = String::from("<html><head><title>/debug/pprof/</title></head><body><h1>/debug/pprof/</h1><ul>");
293 for name in PROFILES {
294 html.push_str(&format!("<li><a href=\"{name}\">{name}</a></li>\n"));
295 }
296 html.push_str("</ul><p>Go runtime profiles are unavailable in Rust. CPU profiling requires the pprof build feature.</p></body></html>");
297 Html(html).into_response()
298}
299
300async fn cmdline() -> Response {
301 text(
303 StatusCode::OK,
304 std::env::args_os()
305 .map(|s| s.to_string_lossy().into_owned())
306 .collect::<Vec<_>>()
307 .join("\0"),
308 )
309}
310
311async fn symbol(OriginalUri(uri): OriginalUri, req: Request) -> Response {
312 let raw = if req.method() == Method::POST {
313 match axum::body::to_bytes(req.into_body(), crate::MAX_REQUEST_BYTES).await {
314 Ok(body) => String::from_utf8_lossy(&body).into_owned(),
315 Err(_) => return text(StatusCode::PAYLOAD_TOO_LARGE, "symbol body too large\n"),
316 }
317 } else {
318 uri.query().unwrap_or_default().to_owned()
319 };
320 #[cfg(not(feature = "pprof"))]
321 {
322 let _ = raw;
323 text(StatusCode::OK, "num_symbols: 1\n")
324 }
325 #[cfg(feature = "pprof")]
326 {
327 static SYMBOLS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
330 let capacity = SYMBOLS.acquire().await.expect("symbol semaphore is never closed");
331 let (active, cancelled) = watch::channel(());
332 let result = tokio::task::spawn_blocking(move || {
333 let _capacity = capacity;
334 let mut output = String::from("num_symbols: 1\n");
335 for pc in raw.split('+') {
336 if cancelled.has_changed().is_err() {
337 break;
338 }
339 let parsed = if let Some(hex) = pc.strip_prefix("0x").or_else(|| pc.strip_prefix("0X")) {
340 usize::from_str_radix(hex, 16)
341 } else if pc.starts_with('0') {
342 usize::from_str_radix(pc, 8)
343 } else {
344 pc.parse()
345 };
346 if let Ok(addr) = parsed
347 && addr != 0
348 {
349 backtrace::resolve(addr.wrapping_add(1) as *mut std::ffi::c_void, |symbol| {
351 if let Some(name) = symbol.name() {
352 output.push_str(&format!("{addr:#x} {name}\n"));
353 }
354 });
355 }
356 }
357 output
358 })
359 .await;
360 drop(active);
361 match result {
362 Ok(output) => text(StatusCode::OK, output),
363 Err(_) => text(StatusCode::INTERNAL_SERVER_ERROR, "Could not resolve symbols\n"),
364 }
365 }
366}
367
368async fn unsupported(axum::extract::Path(name): axum::extract::Path<String>) -> Response {
369 if PROFILES.contains(&name.as_str()) {
370 text(
371 StatusCode::NOT_IMPLEMENTED,
372 format!("{name}: Go runtime profiles are unavailable in Rust\n"),
373 )
374 } else {
375 text(StatusCode::NOT_FOUND, "Unknown profile\n")
376 }
377}
378
379#[cfg(not(all(feature = "pprof", unix)))]
380async fn profile() -> Response {
381 text(
382 StatusCode::NOT_IMPLEMENTED,
383 "CPU profiling requires the cpa-server/pprof build feature on Unix\n",
384 )
385}
386
387#[cfg(all(feature = "pprof", unix))]
388async fn profile(OriginalUri(uri): OriginalUri, req: Request) -> Response {
389 use pprof::protos::Message;
390 let mut seconds = None;
392 if matches!(*req.method(), Method::POST | Method::PUT | Method::PATCH)
393 && req.headers().get(header::CONTENT_TYPE).is_some_and(|v| {
395 v.to_str().unwrap_or_default().split(';').next().unwrap_or_default().trim()
396 .eq_ignore_ascii_case("application/x-www-form-urlencoded")
397 })
398 {
399 let Ok(body) = axum::body::to_bytes(req.into_body(), crate::MAX_REQUEST_BYTES).await else {
400 return text(StatusCode::PAYLOAD_TOO_LARGE, "profile body too large\n");
401 };
402 seconds = crate::access::query_get(&String::from_utf8_lossy(&body), "seconds");
403 }
404 let seconds = seconds
406 .or_else(|| crate::access::query_get(uri.query().unwrap_or_default(), "seconds"))
407 .and_then(|s| String::from_utf8(s).ok())
408 .and_then(|s| s.parse::<i64>().ok())
409 .filter(|n| *n > 0)
410 .unwrap_or(30);
411 let duration = Duration::from_nanos(seconds.wrapping_mul(1_000_000_000).max(0) as u64);
413 static CPU: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1);
415 let Ok(permit) = CPU.try_acquire() else {
416 return text(StatusCode::INTERNAL_SERVER_ERROR, "Could not enable CPU profiling\n");
417 };
418 let started = tokio::task::spawn_blocking(move || {
419 pprof::ProfilerGuardBuilder::default()
420 .frequency(100)
421 .blocklist(&["libc", "libgcc", "pthread", "vdso"])
422 .build()
423 .map(|guard| (guard, permit))
424 })
425 .await;
426 let (guard, permit) = match started {
427 Ok(Ok(started)) => started,
428 _ => return text(StatusCode::INTERNAL_SERVER_ERROR, "Could not enable CPU profiling\n"),
429 };
430 let guard = guard;
432 tokio::time::sleep(duration).await;
434 let bytes = tokio::task::spawn_blocking(move || {
437 let _permit = permit;
438 let report = guard.report().build();
439 drop(guard);
440 report
441 .and_then(|r| r.pprof())
442 .ok()
443 .and_then(|p| p.write_to_bytes().ok())
444 })
445 .await
446 .ok()
447 .flatten();
448 match bytes {
449 Some(bytes) => (
450 [
451 (header::CONTENT_TYPE, "application/octet-stream"),
452 (header::CONTENT_DISPOSITION, "attachment; filename=\"profile\""),
453 (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
454 ],
455 bytes,
456 )
457 .into_response(),
458 None => text(StatusCode::INTERNAL_SERVER_ERROR, "Could not build CPU profile\n"),
459 }
460}
461
462#[cfg(test)]
463mod tests {
464 use super::*;
465 use tokio::io::{AsyncReadExt, AsyncWriteExt};
466
467 async fn symbol_on(stream: &mut tokio::net::TcpStream) {
468 stream
469 .write_all(b"GET /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\n\r\n")
470 .await
471 .unwrap();
472 let mut headers = Vec::new();
473 while !headers.ends_with(b"\r\n\r\n") {
474 headers.push(stream.read_u8().await.unwrap());
475 }
476 assert!(headers.starts_with(b"HTTP/1.1 200"));
477 let mut body = [0; 15];
478 stream.read_exact(&mut body).await.unwrap();
479 assert_eq!(&body, b"num_symbols: 1\n");
480 }
481
482 #[tokio::test]
483 async fn pprof_routes_and_symbols() {
484 let response = router()
485 .oneshot(
486 Request::builder()
487 .method(Method::POST)
488 .uri("/debug/pprof?seconds=2")
489 .body(axum::body::Body::empty())
490 .unwrap(),
491 )
492 .await
493 .unwrap();
494 assert_eq!(response.status(), 307);
495 assert_eq!(response.headers()[header::LOCATION], "/debug/pprof/?seconds=2");
496 for (path, expected) in [("", 200), ("cmdline", 200), ("symbol", 200), ("unknown", 404)] {
497 let response = router()
498 .oneshot(
499 Request::builder()
500 .uri(format!("/debug/pprof/{path}"))
501 .body(axum::body::Body::empty())
502 .unwrap(),
503 )
504 .await
505 .unwrap();
506 assert_eq!(response.status(), expected, "{path}");
507 }
508 for name in ["allocs", "block", "goroutine", "heap", "mutex", "threadcreate", "trace"] {
509 let response = router()
510 .oneshot(
511 Request::builder()
512 .uri(format!("/debug/pprof/{name}"))
513 .body(axum::body::Body::empty())
514 .unwrap(),
515 )
516 .await
517 .unwrap();
518 assert_eq!(response.status(), 501, "{name}");
519 assert!(
520 response.headers()[header::CONTENT_TYPE]
521 .to_str()
522 .unwrap()
523 .starts_with("text/plain")
524 );
525 }
526 let addr = pprof_routes_and_symbols as *const () as usize;
527 for method in [Method::GET, Method::POST] {
528 let raw = format!("{addr:#x}+0x0+garbage");
529 let uri = if method == Method::GET {
530 format!("/debug/pprof/symbol?{raw}")
531 } else {
532 "/debug/pprof/symbol".into()
533 };
534 let response = router()
535 .oneshot(
536 Request::builder()
537 .method(method)
538 .uri(uri)
539 .body(axum::body::Body::from(raw))
540 .unwrap(),
541 )
542 .await
543 .unwrap();
544 let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
545 let body = String::from_utf8(body.to_vec()).unwrap();
546 assert!(body.starts_with("num_symbols: 1\n"));
547 #[cfg(feature = "pprof")]
548 assert!(body.contains("pprof_routes_and_symbols"), "{body}");
549 #[cfg(not(feature = "pprof"))]
550 assert_eq!(body, "num_symbols: 1\n");
551 }
552 #[cfg(not(feature = "pprof"))]
553 {
554 let response = router()
555 .oneshot(
556 Request::builder()
557 .uri("/debug/pprof/profile")
558 .body(axum::body::Body::empty())
559 .unwrap(),
560 )
561 .await
562 .unwrap();
563 assert_eq!(response.status(), 501);
564 let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
565 assert!(String::from_utf8_lossy(&body).contains("cpa-server/pprof"));
566 }
567 }
568
569 #[tokio::test]
570 async fn pprof_listener_reload_failure_and_shutdown() {
571 let dir = std::env::temp_dir().join(format!("pprof-{}", uuid::Uuid::new_v4()));
572 std::fs::create_dir_all(&dir).unwrap();
573 let config = |enabled, addr: &str, legacy| {
574 let pprof = format!("pprof: {{enable: {enabled}, addr: '{addr}'}}");
575 Config::parse(&format!(
576 "oauth: {{auth-dir: '{}'}}\n{}\n",
577 dir.display(),
578 if legacy {
579 pprof
580 } else {
581 format!("observability:\n {pprof}")
582 }
583 ))
584 .unwrap()
585 };
586 let executors = cpa_exec::Executors {
587 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
588 codex: Default::default(),
589 devices: Default::default(),
590 openai: Default::default(),
591 google: Default::default(),
592 };
593 let occupied = TcpListener::bind("127.0.0.1:0").await.unwrap();
594 let addr = occupied.local_addr().unwrap().to_string();
595 let rt = Arc::new(crate::testing::runtime(config(false, &addr, false), vec![], executors));
596 let server = Server::start(&rt);
597 tokio::task::yield_now().await;
598 rt.publish_config(config(true, &addr, false));
599 tokio::task::yield_now().await;
600 drop(occupied);
602 rt.publish_config(config(true, &addr, true));
603 let client = wreq::Client::builder().no_proxy().build().unwrap();
604 let fetch = |addr: String| {
605 let client = client.clone();
606 async move {
607 tokio::time::timeout(Duration::from_secs(7), async {
608 loop {
609 if let Ok(response) = client.get(format!("http://{addr}/debug/pprof/cmdline")).send().await {
610 assert_eq!(response.status(), 200);
611 let expected = std::env::args_os()
612 .map(|s| s.to_string_lossy().into_owned())
613 .collect::<Vec<_>>()
614 .join("\0");
615 assert_eq!(response.text().await.unwrap(), expected);
616 break;
617 }
618 tokio::time::sleep(Duration::from_millis(10)).await;
619 }
620 })
621 .await
622 .unwrap();
623 }
624 };
625 fetch(addr.clone()).await;
626 let mut keep = tokio::net::TcpStream::connect(&addr).await.unwrap();
628 symbol_on(&mut keep).await;
629 rt.publish_config(config(true, &addr, false));
630 tokio::task::yield_now().await;
631 symbol_on(&mut keep).await;
632 let started = tokio::time::Instant::now();
635 for _ in 0..9 {
636 let (accepted, observed) = tokio::sync::oneshot::channel();
637 #[cfg(unix)]
638 let error = std::io::Error::from_raw_os_error(libc::EMFILE);
639 #[cfg(windows)]
640 let error = std::io::Error::from_raw_os_error(10024);
641 server.accept_errors.send((error, accepted)).unwrap();
642 observed.await.unwrap();
643 symbol_on(&mut keep).await;
644 }
645 assert!(
646 started.elapsed() >= Duration::from_secs(1),
647 "accept errors did not back off"
648 );
649 rt.publish_config(config(false, &addr, false));
651 tokio::time::timeout(Duration::from_millis(500), async {
652 while tokio::net::TcpStream::connect(&addr).await.is_ok() {
653 tokio::task::yield_now().await;
654 }
655 })
656 .await
657 .unwrap();
658 rt.publish_config(config(true, &addr, false));
659 fetch(addr.clone()).await;
660 let mut active = tokio::net::TcpStream::connect(&addr).await.unwrap();
663 active.write_all(b"POST /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\n").await.unwrap();
664 let mut interim = [0; 25];
665 active.read_exact(&mut interim).await.unwrap();
666 assert_eq!(&interim, b"HTTP/1.1 100 Continue\r\n\r\n");
667 let replacement = TcpListener::bind("127.0.0.1:0").await.unwrap();
668 let stale = replacement.local_addr().unwrap().to_string();
669 drop(replacement);
670 rt.publish_config(config(true, &stale, false));
671 tokio::time::timeout(Duration::from_secs(5), async {
672 while tokio::net::TcpStream::connect(&addr).await.is_ok() {
673 tokio::time::sleep(Duration::from_millis(10)).await;
674 }
675 })
676 .await
677 .unwrap();
678 tokio::time::sleep(STOP_TIMEOUT + Duration::from_millis(100)).await;
679 assert!(tokio::net::TcpStream::connect(&stale).await.is_err());
680 let replacement = crate::listener::bind("", 0).unwrap();
681 let port = replacement.local_addr().unwrap().port();
682 let ipv6 = replacement.local_addr().unwrap().is_ipv6();
683 let next = format!("127.0.0.1:{port}");
684 let wildcard = format!(":{port}");
685 drop(replacement);
686 rt.publish_config(config(true, &wildcard, false));
687 fetch(next.clone()).await;
689 active.write_all(b"0x0").await.unwrap();
690 let mut completed = String::new();
691 tokio::time::timeout(Duration::from_secs(5), active.read_to_string(&mut completed))
692 .await
693 .unwrap()
694 .unwrap();
695 assert!(completed.starts_with("HTTP/1.1 200"), "{completed}");
696 assert!(completed.contains("num_symbols: 1"), "{completed}");
697 assert!(tokio::net::TcpStream::connect(&addr).await.is_err());
698 assert!(tokio::net::TcpStream::connect(&stale).await.is_err());
699 if ipv6 {
700 fetch(format!("[::1]:{port}")).await;
701 }
702 rt.publish_config(config(false, &next, false));
703 tokio::time::timeout(Duration::from_secs(5), async {
704 while tokio::net::TcpStream::connect(&next).await.is_ok() {
705 tokio::time::sleep(Duration::from_millis(10)).await;
706 }
707 })
708 .await
709 .unwrap();
710 rt.publish_config(config(true, &next, false));
711 fetch(next.clone()).await;
712 let mut keep = tokio::net::TcpStream::connect(&next).await.unwrap();
714 symbol_on(&mut keep).await;
715 let mut held = tokio::net::TcpStream::connect(&next).await.unwrap();
716 held.write_all(b"POST /debug/pprof/symbol HTTP/1.1\r\nHost: localhost\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\n").await.unwrap();
717 let mut interim = [0; 25];
718 tokio::time::timeout(Duration::from_secs(5), held.read_exact(&mut interim))
719 .await
720 .unwrap()
721 .unwrap();
722 assert_eq!(&interim, b"HTTP/1.1 100 Continue\r\n\r\n");
723 let (accepted, observed) = tokio::sync::oneshot::channel();
724 server
725 .accept_errors
726 .send((std::io::Error::other("terminal accept failure"), accepted))
727 .unwrap();
728 observed.await.unwrap();
729 symbol_on(&mut keep).await;
730 tokio::time::sleep(Duration::from_millis(50)).await;
731 assert!(tokio::net::TcpStream::connect(&next).await.is_err());
732 rt.publish_config(config(true, &next, false));
735 let mut rest = Vec::new();
737 let read = tokio::time::timeout(Duration::from_secs(3), keep.read_to_end(&mut rest)).await;
738 assert!(matches!(read, Ok(Ok(0))), "{read:?}");
739 assert!(tokio::net::TcpStream::connect(&next).await.is_err());
741 held.write_all(b"0x0").await.unwrap();
745 let mut completed = String::new();
746 tokio::time::timeout(Duration::from_secs(5), held.read_to_string(&mut completed))
747 .await
748 .unwrap()
749 .unwrap();
750 assert!(completed.starts_with("HTTP/1.1 200"), "{completed}");
751 fetch(next.clone()).await;
752 server.shutdown().await;
753 assert!(tokio::net::TcpStream::connect(&next).await.is_err());
754 std::fs::remove_dir_all(dir).unwrap();
755 }
756
757 #[cfg(all(feature = "pprof", unix))]
758 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
759 async fn pprof_cpu_profile_is_decodable_and_exclusive() {
760 use pprof::protos::Message;
761 let request = || {
762 Request::builder()
763 .uri("/debug/pprof/profile?seconds=1")
764 .body(axum::body::Body::empty())
765 .unwrap()
766 };
767 let mut first = Box::pin(router().oneshot(request()));
768 assert!(
769 tokio::time::timeout(Duration::from_millis(30), &mut first)
770 .await
771 .is_err()
772 );
773 let first = tokio::spawn(first);
774 let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
777 let stopped = stop.clone();
778 let workload = std::thread::spawn(move || {
779 while !stopped.load(std::sync::atomic::Ordering::Relaxed) {
780 std::hint::black_box((0u64..1000).fold(0u64, |sum, n| sum.wrapping_add(n)));
781 }
782 });
783 let second = router().oneshot(request()).await.unwrap();
784 assert_eq!(second.status(), 500);
785 let first = first.await.unwrap().unwrap();
786 stop.store(true, std::sync::atomic::Ordering::Relaxed);
787 workload.join().unwrap();
788 assert_eq!(first.status(), 200);
789 assert_eq!(first.headers()[header::CONTENT_TYPE], "application/octet-stream");
790 let bytes = axum::body::to_bytes(first.into_body(), usize::MAX).await.unwrap();
791 let decoded = pprof::protos::Profile::parse_from_bytes(&bytes).unwrap();
792 assert!(decoded.duration_nanos > 0);
793 assert!(!decoded.sample.is_empty(), "profile did not sample the CPU workload");
794 let mut pending = Box::pin(router().oneshot(request()));
796 assert!(
797 tokio::time::timeout(Duration::from_millis(30), &mut pending)
798 .await
799 .is_err()
800 );
801 drop(pending);
802 let types = [
803 "application/x-www-form-urlencoded",
804 " Application/X-Www-Form-Urlencoded ; charset=utf-8",
805 ];
806 for (method, content_type) in [Method::POST, Method::PUT, Method::PATCH]
807 .into_iter()
808 .flat_map(|m| types.map(|t| (m.clone(), t)))
809 {
810 let response = tokio::time::timeout(
811 Duration::from_secs(5),
812 router().oneshot(
813 Request::builder()
814 .method(method)
815 .uri("/debug/pprof/profile?seconds=30")
816 .header(header::CONTENT_TYPE, content_type)
817 .body(axum::body::Body::from("seconds=1"))
818 .unwrap(),
819 ),
820 )
821 .await
822 .unwrap()
823 .unwrap();
824 assert_eq!(response.status(), 200);
825 }
826 }
827}