use axum::extract::ws::{Message, WebSocket};
use axum::extract::{Path as AxumPath, State, WebSocketUpgrade};
use axum::response::IntoResponse;
use tokio::sync::broadcast;
use super::types::*;
pub(super) async fn ws_global(
State(state): State<AppState>,
ws: WebSocketUpgrade,
) -> impl IntoResponse {
let rx = state.event_tx.subscribe();
ws.on_upgrade(move |socket| handle_ws(socket, rx, None))
}
pub(super) async fn ws_agent(
State(state): State<AppState>,
AxumPath(id): AxumPath<String>,
ws: WebSocketUpgrade,
) -> impl IntoResponse {
let rx = state.event_tx.subscribe();
ws.on_upgrade(move |socket| handle_ws(socket, rx, Some(id)))
}
async fn handle_ws(
mut socket: WebSocket,
mut rx: broadcast::Receiver<ServerEvent>,
filter_run_id: Option<String>,
) {
loop {
tokio::select! {
biased;
event = rx.recv() => {
match event {
Ok(ev) => {
if let Some(ref filter) = filter_run_id
&& ev.run_id() != filter
{
continue;
}
let json = serde_json::to_string(&ev)
.expect("ServerEvent serialization must not fail");
if socket.send(Message::Text(json.into())).await.is_err() {
break;
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!("WebSocket subscriber lagged by {} events", n);
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
msg = socket.recv() => {
match msg {
Some(Ok(Message::Close(_))) | None => break,
_ => {} }
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::Router;
use axum::routing::get;
use std::sync::Arc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use crate::config::Config;
use crate::test_support::with_tracing;
struct WsTestClient {
stream: TcpStream,
}
fn assert_handshake_byte_read(n: usize) {
assert_ne!(n, 0, "connection closed before handshake completed");
}
#[test]
#[should_panic(expected = "connection closed before handshake completed")]
fn assert_handshake_byte_read_panics_on_zero() {
assert_handshake_byte_read(0);
}
fn assert_handshake_101(response: &str) {
#[rustfmt::skip]
assert!(response.starts_with("HTTP/1.1 101"), "expected 101 Switching Protocols, got: {response}");
}
#[test]
#[should_panic(expected = "expected 101 Switching Protocols, got: HTTP/1.1 404 Not Found")]
fn assert_handshake_101_panics_on_non_101() {
assert_handshake_101("HTTP/1.1 404 Not Found");
}
impl WsTestClient {
async fn connect(addr: std::net::SocketAddr, path: &str) -> Self {
let mut stream = TcpStream::connect(addr).await.unwrap();
let request = format!(
"GET {path} HTTP/1.1\r\n\
Host: {addr}\r\n\
Connection: Upgrade\r\n\
Upgrade: websocket\r\n\
Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\
Sec-WebSocket-Version: 13\r\n\
\r\n"
);
stream.write_all(request.as_bytes()).await.unwrap();
let mut buf = Vec::new();
let mut byte = [0u8; 1];
loop {
let n = stream.read(&mut byte).await.unwrap();
assert_handshake_byte_read(n);
buf.push(byte[0]);
if buf.ends_with(b"\r\n\r\n") {
break;
}
}
let response = String::from_utf8_lossy(&buf);
assert_handshake_101(&response);
Self { stream }
}
async fn send_text(&mut self, text: &str) {
self.send_frame(0x1, text.as_bytes()).await;
}
async fn send_close(&mut self) {
self.send_frame(0x8, &[]).await;
}
async fn send_frame(&mut self, opcode: u8, payload: &[u8]) {
let mut frame = vec![0x80 | opcode];
let mask: [u8; 4] = [0x12, 0x34, 0x56, 0x78];
let len = payload.len();
if len < 126 {
frame.push(0x80 | len as u8);
} else {
frame.push(0x80 | 126);
frame.push((len >> 8) as u8);
frame.push(len as u8);
}
frame.extend_from_slice(&mask);
for (i, b) in payload.iter().enumerate() {
frame.push(b ^ mask[i % 4]);
}
self.stream.write_all(&frame).await.unwrap();
}
async fn recv_frame(&mut self) -> (u8, Vec<u8>) {
let mut header = [0u8; 2];
self.stream.read_exact(&mut header).await.unwrap();
let opcode = header[0] & 0x0f;
let mut len = (header[1] & 0x7f) as usize;
if len == 126 {
let mut ext = [0u8; 2];
self.stream.read_exact(&mut ext).await.unwrap();
len = u16::from_be_bytes(ext) as usize;
} else if len == 127 {
let mut ext = [0u8; 8];
self.stream.read_exact(&mut ext).await.unwrap();
len = u64::from_be_bytes(ext) as usize;
}
let mut payload = vec![0u8; len];
if len > 0 {
self.stream.read_exact(&mut payload).await.unwrap();
}
(opcode, payload)
}
async fn recv_eof(&mut self) -> Option<u8> {
let mut byte = [0u8; 1];
match self.stream.read(&mut byte).await {
Ok(0) => None,
Ok(_) => Some(byte[0]),
Err(_) => None,
}
}
}
fn test_state() -> AppState {
let (tx, _) = broadcast::channel(64);
AppState {
config: Arc::new(Config::default()),
event_tx: tx,
control: crate::commands::serve::testutil::no_daemon_client(),
mcp: crate::commands::serve::mcp::McpAdmin::default(),
limits: Default::default(),
}
}
async fn spawn_test_server_with_shutdown(
state: AppState,
) -> (
std::net::SocketAddr,
tokio::sync::oneshot::Sender<()>,
tokio::task::JoinHandle<()>,
) {
let app = Router::new()
.route("/ws", get(ws_global))
.route("/ws/agents/{id}", get(ws_agent))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let handle = tokio::spawn(async move {
let _ = axum::serve(listener, app)
.with_graceful_shutdown(async {
let _ = shutdown_rx.await;
})
.await;
});
(addr, shutdown_tx, handle)
}
async fn spawn_test_server(state: AppState) -> std::net::SocketAddr {
let (addr, shutdown_tx, _handle) = spawn_test_server_with_shutdown(state).await;
std::mem::forget(shutdown_tx);
addr
}
fn assert_text_frame(opcode: u8) {
assert_eq!(opcode, 0x1, "expected a text frame");
}
#[test]
#[should_panic(expected = "expected a text frame")]
fn assert_text_frame_panics_on_non_text_opcode() {
assert_text_frame(0x2);
}
#[tokio::test]
async fn ws_global_relays_broadcast_event_to_client() {
let state = test_state();
let tx = state.event_tx.clone();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-1".to_string(),
line: "hello".to_string(),
})
.unwrap();
let (opcode, payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("timed out waiting for event frame");
assert_text_frame(opcode);
let text = String::from_utf8(payload).unwrap();
assert!(text.contains("\"type\":\"log\""));
assert!(text.contains("\"run_id\":\"run-1\""));
client.send_close().await;
}
#[tokio::test]
async fn ws_global_relays_large_event_using_64bit_extended_length() {
let state = test_state();
let tx = state.event_tx.clone();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let huge_line = "x".repeat(70_000);
tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-huge".to_string(),
line: huge_line.clone(),
})
.unwrap();
let (opcode, payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("timed out waiting for large event frame");
assert_eq!(opcode, 0x1);
let text = String::from_utf8(payload).unwrap();
assert!(text.contains(&huge_line));
client.send_close().await;
}
#[tokio::test]
async fn ws_test_client_send_frame_encodes_medium_length_payload() {
let state = test_state();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
let payload = "y".repeat(200);
client.send_frame(0x1, payload.as_bytes()).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
client.send_close().await;
}
#[tokio::test]
async fn ws_agent_filters_events_by_run_id() {
let state = test_state();
let tx = state.event_tx.clone();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws/agents/run-match").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-other".to_string(),
line: "skip me".to_string(),
})
.unwrap();
tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
line: "deliver me".to_string(),
})
.unwrap();
let (opcode, payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("timed out waiting for event frame");
assert_eq!(opcode, 0x1);
let text = String::from_utf8(payload).unwrap();
assert!(text.contains("\"run_id\":\"run-match\""));
assert!(!text.contains("run-other"));
client.send_close().await;
}
#[tokio::test]
async fn ws_agent_filter_matches_run_id_for_every_event_variant() {
let state = test_state();
let tx = state.event_tx.clone();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws/agents/run-match").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let events = vec![
ServerEvent::AgentStatus {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
status: "running".to_string(),
stage: "plan".to_string(),
iteration: 1,
tool_calls: 0,
accepts_messages: true,
},
ServerEvent::ContextUpdate {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
total_tokens: 10,
max_tokens: 100,
},
ServerEvent::InteractionNeeded {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
request: serde_json::Value::Null,
},
ServerEvent::AgentSpawned {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
parent_id: None,
blueprint: "bp".to_string(),
},
ServerEvent::AgentCompleted {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
status: "complete".to_string(),
result: None,
},
ServerEvent::Tokens {
agent_id: "a".to_string(),
run_id: "run-match".to_string(),
prompt_tokens: 1,
completion_tokens: 2,
cached_tokens: 0,
cache_write_tokens: 0,
},
];
for ev in events {
tx.send(ev).unwrap();
let (opcode, payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("timed out waiting for event frame");
assert_eq!(opcode, 0x1);
let text = String::from_utf8(payload).unwrap();
assert!(text.contains("\"run_id\":\"run-match\""));
}
client.send_close().await;
}
fn assert_clean_eof_after_close(eof: Option<u8>) {
assert_eq!(eof, None, "expected clean EOF after server processed close");
}
#[test]
#[should_panic(expected = "expected clean EOF after server processed close")]
fn assert_clean_eof_after_close_panics_on_some() {
assert_clean_eof_after_close(Some(0x42));
}
#[tokio::test]
async fn ws_global_closes_on_client_close_frame() {
let state = test_state();
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
client.send_text("ignored client message").await;
client.send_close().await;
let eof = tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_eof())
.await
.expect("timed out waiting for server to close the connection");
assert_clean_eof_after_close(eof);
}
#[tokio::test]
async fn ws_global_lagged_receiver_does_not_crash_connection() {
with_tracing(|| {});
let (tx, _) = broadcast::channel::<ServerEvent>(2);
let state = AppState {
config: Arc::new(Config::default()),
event_tx: tx.clone(),
control: crate::commands::serve::testutil::no_daemon_client(),
mcp: crate::commands::serve::mcp::McpAdmin::default(),
limits: Default::default(),
};
let addr = spawn_test_server(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
for i in 0..20 {
let _ = tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-1".to_string(),
line: format!("line-{i}"),
});
}
let (opcode, _payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("connection should still be alive after a lag");
assert_eq!(opcode, 0x1);
client.send_close().await;
}
#[tokio::test]
async fn ws_global_breaks_on_abrupt_tcp_close_without_ws_close_frame() {
let state = test_state();
let addr = spawn_test_server(state).await;
let client = WsTestClient::connect(addr, "/ws").await;
drop(client.stream);
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let mut second = WsTestClient::connect(addr, "/ws").await;
second.send_close().await;
}
#[tokio::test]
async fn ws_global_breaks_when_send_fails_after_abrupt_client_close() {
let state = test_state();
let tx = state.event_tx.clone();
let addr = spawn_test_server(state).await;
let client = WsTestClient::connect(addr, "/ws").await;
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
for i in 0..100 {
let _ = tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-1".to_string(),
line: format!("pre-drop flood event {i}"),
});
}
drop(client.stream);
for i in 0..50 {
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
let _ = tx.send(ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-1".to_string(),
line: format!("post-drop event {i}"),
});
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let mut second = WsTestClient::connect(addr, "/ws").await;
second.send_close().await;
}
fn assert_closed_after_channel_closed(eof: Option<u8>) {
assert_eq!(eof, None, "server should close after channel Closed");
}
#[test]
#[should_panic(expected = "server should close after channel Closed")]
fn assert_closed_after_channel_closed_panics_on_some() {
assert_closed_after_channel_closed(Some(0x1));
}
#[tokio::test]
async fn handle_ws_breaks_on_closed_channel_via_server_shutdown() {
let (tx, _) = broadcast::channel::<ServerEvent>(16);
let state = AppState {
config: Arc::new(Config::default()),
event_tx: tx.clone(),
control: crate::commands::serve::testutil::no_daemon_client(),
mcp: crate::commands::serve::mcp::McpAdmin::default(),
limits: Default::default(),
};
let (addr, shutdown_tx, handle) = spawn_test_server_with_shutdown(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
drop(tx);
let _ = shutdown_tx.send(());
tokio::time::timeout(std::time::Duration::from_secs(5), handle)
.await
.expect("server did not shut down in time")
.expect("server panicked");
let eof = tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_eof())
.await
.expect("timed out waiting for server to close after channel closed");
assert_closed_after_channel_closed(eof);
}
#[tokio::test]
async fn spawn_test_server_axum_serve_returns_on_graceful_shutdown() {
let state = test_state();
let (addr, shutdown_tx, handle) = spawn_test_server_with_shutdown(state).await;
let mut client = WsTestClient::connect(addr, "/ws").await;
client.send_close().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let _ = shutdown_tx.send(());
tokio::time::timeout(std::time::Duration::from_secs(5), handle)
.await
.expect("timed out waiting for server to shut down")
.unwrap();
}
fn assert_text_opcode(opcode: u8) {
assert_eq!(opcode, 0x1, "expected text opcode");
}
#[test]
#[should_panic(expected = "expected text opcode")]
fn assert_text_opcode_panics_on_non_text() {
assert_text_opcode(0x2);
}
fn assert_empty_payload(payload: &[u8]) {
assert!(payload.is_empty(), "expected empty payload");
}
#[test]
#[should_panic(expected = "expected empty payload")]
fn assert_empty_payload_panics_on_nonempty() {
assert_empty_payload(&[1, 2, 3]);
}
#[tokio::test]
async fn recv_frame_handles_zero_length_payload() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
let frame = [0x81u8, 0x00];
let _ = sock.write_all(&frame).await;
let _ = sock.shutdown().await;
});
let stream = TcpStream::connect(addr).await.unwrap();
let mut client = WsTestClient { stream };
let (opcode, payload) =
tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_frame())
.await
.expect("timed out waiting for zero-length frame");
assert_text_opcode(opcode);
assert_empty_payload(&payload);
}
fn assert_none_on_clean_eof(result: Option<u8>) {
assert_eq!(result, None, "expected None on clean EOF");
}
#[test]
#[should_panic(expected = "expected None on clean EOF")]
fn assert_none_on_clean_eof_panics_on_some() {
assert_none_on_clean_eof(Some(0x1));
}
#[tokio::test]
async fn recv_eof_returns_none_on_clean_server_shutdown() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut conn, _) = listener.accept().await.unwrap();
let mut buf = [0u8; 256];
let _ = conn.read(&mut buf).await;
conn.shutdown().await.unwrap();
});
let mut stream = TcpStream::connect(addr).await.unwrap();
let _ = stream.write_all(b"hi").await;
let mut client = WsTestClient { stream };
let result = tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_eof())
.await
.expect("timed out");
assert_none_on_clean_eof(result);
}
fn assert_some_byte_arrived(result: Option<u8>) {
assert_eq!(result, Some(0x42), "expected the sent byte");
}
#[test]
#[should_panic(expected = "expected the sent byte")]
fn assert_some_byte_arrived_panics_on_mismatch() {
assert_some_byte_arrived(Some(0x99));
}
#[tokio::test]
async fn recv_eof_returns_some_when_byte_arrives() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut conn, _) = listener.accept().await.unwrap();
let _ = conn.write_all(&[0x42u8]).await;
let _ = conn.shutdown().await;
});
let stream = TcpStream::connect(addr).await.unwrap();
let mut client = WsTestClient { stream };
let result = tokio::time::timeout(std::time::Duration::from_secs(5), client.recv_eof())
.await
.expect("timed out");
assert_some_byte_arrived(result);
}
#[test]
fn server_event_run_id_extraction_agent_status() {
let ev = ServerEvent::AgentStatus {
agent_id: "coder".to_string(),
run_id: "run-123".to_string(),
status: "running".to_string(),
stage: "plan".to_string(),
iteration: 1,
tool_calls: 0,
accepts_messages: true,
};
assert_eq!(ev.run_id(), "run-123");
}
#[test]
fn server_event_run_id_extraction_context_update() {
let ev = ServerEvent::ContextUpdate {
agent_id: "a".to_string(),
run_id: "run-ctx".to_string(),
total_tokens: 100,
max_tokens: 200000,
};
assert_eq!(ev.run_id(), "run-ctx");
}
#[test]
fn server_event_run_id_extraction_all_variants() {
let variants: Vec<(ServerEvent, &str)> = vec![
(
ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "run-log".to_string(),
line: "hi".to_string(),
},
"run-log",
),
(
ServerEvent::InteractionNeeded {
agent_id: "a".to_string(),
run_id: "run-int".to_string(),
request: serde_json::Value::Null,
},
"run-int",
),
(
ServerEvent::AgentSpawned {
agent_id: "a".to_string(),
run_id: "run-spawn".to_string(),
parent_id: None,
blueprint: "bp".to_string(),
},
"run-spawn",
),
(
ServerEvent::AgentCompleted {
agent_id: "a".to_string(),
run_id: "run-done".to_string(),
status: "complete".to_string(),
result: None,
},
"run-done",
),
(
ServerEvent::Tokens {
agent_id: "a".to_string(),
run_id: "run-tok".to_string(),
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
cache_write_tokens: 0,
},
"run-tok",
),
];
for (ev, expected) in variants {
assert_eq!(ev.run_id(), expected);
}
}
#[test]
fn server_event_filter_matching() {
let filter = "run-123".to_string();
let matching = ServerEvent::AgentStatus {
agent_id: "a".to_string(),
run_id: "run-123".to_string(),
status: "running".to_string(),
stage: "plan".to_string(),
iteration: 1,
tool_calls: 0,
accepts_messages: true,
};
let non_matching = ServerEvent::AgentStatus {
agent_id: "a".to_string(),
run_id: "run-456".to_string(),
status: "running".to_string(),
stage: "plan".to_string(),
iteration: 1,
tool_calls: 0,
accepts_messages: true,
};
assert_eq!(matching.run_id(), filter);
assert_ne!(non_matching.run_id(), filter);
}
#[test]
fn server_event_serializes_to_json() {
let ev = ServerEvent::AgentStatus {
agent_id: "coder".to_string(),
run_id: "run-ws".to_string(),
status: "running".to_string(),
stage: "plan".to_string(),
iteration: 3,
tool_calls: 0,
accepts_messages: false,
};
let json = serde_json::to_string(&ev).unwrap();
assert!(json.contains("\"type\":\"agent_status\""));
assert!(json.contains("\"run_id\":\"run-ws\""));
}
#[test]
fn broadcast_channel_creation() {
let (tx, _rx) = broadcast::channel::<ServerEvent>(16);
let ev = ServerEvent::Log {
agent_id: "a".to_string(),
run_id: "r".to_string(),
line: "test".to_string(),
};
assert!(tx.send(ev).is_ok());
}
fn assert_none_on_connection_reset(result: Option<u8>) {
assert_eq!(result, None, "expected None on connection reset");
}
#[test]
#[should_panic(expected = "expected None on connection reset")]
fn assert_none_on_connection_reset_panics_on_some() {
assert_none_on_connection_reset(Some(0x1));
}
#[tokio::test]
async fn recv_eof_returns_none_on_io_error() {
use std::time::Duration;
use tokio::net::TcpSocket;
let server_sock = TcpSocket::new_v4().unwrap();
server_sock.bind("127.0.0.1:0".parse().unwrap()).unwrap();
let addr = server_sock.local_addr().unwrap();
let listener = server_sock.listen(1).unwrap();
tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
#[allow(deprecated)]
stream.set_linger(Some(Duration::from_secs(0))).unwrap();
});
let stream = TcpStream::connect(addr).await.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let mut client = WsTestClient { stream };
let result = tokio::time::timeout(std::time::Duration::from_secs(2), client.recv_eof())
.await
.expect("timed out waiting for recv_eof on RST");
assert_none_on_connection_reset(result);
}
}