use super::*;
use axum::{body::Body, http::Request};
use dbgflow_protocol::{
CURRENT_PROJECTION_VERSION, EventContext, FlowId, Payload, PayloadEncoding, PayloadFlags,
ProcessId, RecordingId, SpanId, ThreadId, Timestamp,
};
use dbgflow_runtime::{Recorder, RecorderConfig};
use dbgflow_storage::{RecordingOptions, RecordingWriter, RetainedPrefix, RetentionPolicy};
use http_body_util::BodyExt as _;
use serde_json::Value;
use tempfile::TempDir;
use tower::ServiceExt as _;
fn request(uri: &str) -> axum::http::request::Builder {
Request::get(uri).header(HOST, "localhost:0")
}
fn event(recording_id: RecordingId, sequence: u64) -> EventEnvelope {
EventEnvelope::new(
recording_id,
Sequence::new(sequence),
Timestamp::new(sequence * 10, None),
EventContext::new(ProcessId::new(7), ThreadId::new(9)),
if sequence == 4 {
EventKind::Gap
} else {
EventKind::Checkpoint
},
)
}
fn recording_fixture() -> (TempDir, PathBuf) {
let temp = TempDir::new().unwrap();
let path = temp.path().join("server.dbgflow");
let id = RecordingId::from_u128(0x42);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=5 {
writer.append(&event(id, sequence)).unwrap();
}
writer.finish().unwrap();
(temp, path)
}
fn rolling_recording_fixture() -> (TempDir, PathBuf, RetainedPrefix) {
let temp = TempDir::new().unwrap();
let path = temp.path().join("rolling-server.dbgflow");
let id = RecordingId::from_u128(0x44);
let mut writer = RecordingWriter::create(
&path,
id,
RecordingOptions {
max_chunk_bytes: 100,
max_recording_bytes: Some(1_000),
retention_policy: RetentionPolicy::DeleteOldest,
..RecordingOptions::default()
},
)
.unwrap();
for sequence in 1..=100 {
let mut envelope = event(id, sequence);
envelope.kind = EventKind::Checkpoint;
writer.append(&envelope).unwrap();
}
writer.flush().unwrap();
let retained = writer.retained_prefix();
assert!(retained.history_truncated);
assert!(retained.first_sequence.is_some());
assert!(retained.removed_through_sequence.is_some());
writer.finish().unwrap();
(temp, path, retained)
}
async fn json(response: Response) -> Value {
let bytes = response.into_body().collect().await.unwrap().to_bytes();
serde_json::from_slice(&bytes).unwrap()
}
async fn wait_until(mut predicate: impl FnMut() -> bool, message: &str) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
while !predicate() {
assert!(tokio::time::Instant::now() < deadline, "{message}");
tokio::time::sleep(Duration::from_millis(5)).await;
}
}
fn sse_ids(text: &str) -> Vec<u64> {
text.lines()
.filter_map(|line| line.strip_prefix("id: "))
.map(|value| value.parse().unwrap())
.collect()
}
#[tokio::test]
async fn negotiates_versions_and_reports_durable_recording() {
let (_temp, path) = recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
let response = app
.clone()
.oneshot(request("/api/v1/capabilities").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = json(response).await;
assert_eq!(body["api_version"], 1);
assert_eq!(body["schema_version"], CURRENT_SCHEMA_VERSION);
assert_eq!(body["format_version"], CURRENT_FORMAT_VERSION);
assert_eq!(body["transports"], serde_json::json!(["sse"]));
assert_eq!(body["cursor"]["semantics"], "exclusive");
assert_eq!(body["cursor"]["durability"], "synced_frames");
assert_eq!(
body["pagination"]["max_response_bytes"],
ServerConfig::default().max_page_bytes
);
assert_eq!(
body["pagination"]["directions"],
serde_json::json!(["forward", "backward"])
);
assert_eq!(body["retention"]["rolling"], true);
assert_eq!(body["retention"]["notice_event"], "dbgflow.retention");
assert_eq!(
body["replay"]["checkpoint_projection_version"],
CURRENT_PROJECTION_VERSION
);
assert_eq!(body["replay"]["endpoint"], "/api/v1/replay");
let response = app
.oneshot(request("/api/v1/recording").body(Body::empty()).unwrap())
.await
.unwrap();
let body = json(response).await;
assert_eq!(body["recording_id"], format!("{:032x}", 0x42_u128));
assert_eq!(body["state"], "complete");
assert_eq!(body["activity"], "complete");
assert_eq!(body["committed"]["event_count"], 5);
assert_eq!(body["committed"]["last_cursor"], "5");
assert_eq!(
body["durable"]["retained_prefix"]["history_truncated"],
false
);
assert_eq!(body["durable"]["retained_prefix"]["generation"], "0");
}
#[tokio::test]
async fn replay_returns_nearest_typed_checkpoint_and_bounded_deltas() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("replay-server.dbgflow");
let id = RecordingId::from_u128(0x51);
let mut recorder = Recorder::start(
&path,
id,
RecordingOptions::default(),
RecorderConfig::default(),
)
.unwrap();
let sink = recorder.sink();
let mut context = EventContext::new(ProcessId::new(7), ThreadId::new(9));
context.flow_id = Some(FlowId::new(2));
context.span_id = Some(SpanId::new(3));
for kind in [EventKind::SpanOpen, EventKind::SpanEnter] {
let mut envelope =
EventEnvelope::new(id, Sequence::new(0), Timestamp::new(1, None), context, kind);
if kind == EventKind::SpanOpen {
envelope.payload = Some(Payload::new(PayloadEncoding::Utf8, b"request".to_vec()));
}
sink.try_record(envelope).unwrap();
}
recorder.flush().unwrap();
sink.try_record(EventEnvelope::new(
id,
Sequence::new(0),
Timestamp::new(2, None),
context,
EventKind::Log,
))
.unwrap();
recorder.flush().unwrap();
let app = Server::router(ServerConfig::default(), &path).unwrap();
let response = app
.oneshot(
request("/api/v1/replay?cursor=3&limit=10")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = json(response).await;
assert_eq!(body["requested_cursor"], "3");
assert_eq!(body["resolved_cursor"], "3");
assert_eq!(body["checkpoint"]["durable_cursor"], "3");
assert_eq!(body["checkpoint"]["projection_version"], 1);
assert_eq!(body["checkpoint"]["active_spans"][0]["enter_depth"], 1);
assert_eq!(body["checkpoint"]["active_spans"][0]["opened_cursor"], "1");
assert_eq!(body["deltas"], serde_json::json!([]));
assert_eq!(body["exact"], true);
recorder.shutdown().unwrap();
}
#[tokio::test]
async fn replay_pages_deltas_and_never_overstates_exactness() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("replay-page.dbgflow");
let id = RecordingId::from_u128(0x52);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=4 {
writer.append(&event(id, sequence)).unwrap();
}
writer.finish().unwrap();
let app = Server::router(ServerConfig::default(), &path).unwrap();
let body = json(
app.oneshot(
request("/api/v1/replay?cursor=99&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(body["requested_cursor"], "99");
assert_eq!(body["resolved_cursor"], "4");
assert_eq!(body["checkpoint"], Value::Null);
assert_eq!(body["deltas"].as_array().unwrap().len(), 2);
assert_eq!(body["has_more"], true);
assert_eq!(body["next_cursor"], "2");
assert_eq!(body["exact"], false);
}
#[tokio::test]
async fn replay_reconstructs_old_boundary_beyond_checkpoint_and_client_windows() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("replay-arbitrary-old.dbgflow");
let id = RecordingId::from_u128(0x5201);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=12_500 {
let mut envelope = event(id, sequence);
envelope.kind = EventKind::Checkpoint;
writer.append(&envelope).unwrap();
if sequence % 500 == 0 {
writer.flush().unwrap();
writer
.publish_checkpoint(&ProjectionCheckpoint {
projection_version: CURRENT_PROJECTION_VERSION,
recording_id: id,
durable_cursor: Sequence::new(sequence),
complete: true,
active_spans: vec![],
known_spans: vec![],
})
.unwrap();
}
}
writer.finish().unwrap();
let app = Server::router(ServerConfig::default(), &path).unwrap();
let body = json(
app.oneshot(
request("/api/v1/replay?cursor=12500&boundary=1000&limit=5000")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(body["checkpoint"]["durable_cursor"], "1000");
assert_eq!(body["checkpoint"]["complete"], true);
assert_eq!(body["deltas"].as_array().unwrap().len(), 5_000);
assert_eq!(body["deltas"][0]["sequence"], "1001");
assert_eq!(body["deltas"][4999]["sequence"], "6000");
assert_eq!(body["has_more"], true);
assert_eq!(body["next_cursor"], "6000");
let app = Server::router(ServerConfig::default(), &path).unwrap();
let final_page = json(
app.oneshot(
request("/api/v1/replay?cursor=12500&boundary=1000&after=6000&limit=5000")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(final_page["checkpoint"]["durable_cursor"], "1000");
assert_eq!(final_page["deltas"].as_array().unwrap().len(), 5_000);
assert_eq!(final_page["deltas"][0]["sequence"], "6001");
assert_eq!(final_page["deltas"][4999]["sequence"], "11000");
assert_eq!(final_page["has_more"], true);
let app = Server::router(ServerConfig::default(), &path).unwrap();
let final_page = json(
app.oneshot(
request("/api/v1/replay?cursor=12500&boundary=1000&after=11000&limit=5000")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(final_page["deltas"].as_array().unwrap().len(), 1_500);
assert_eq!(final_page["deltas"][0]["sequence"], "11001");
assert_eq!(final_page["deltas"][1499]["sequence"], "12500");
assert_eq!(final_page["exact"], true);
}
#[tokio::test]
async fn replay_pagination_keeps_explicit_checkpoint_boundary_stable() {
use dbgflow_protocol::{ProjectedSpan, ProjectionCheckpoint};
let temp = TempDir::new().unwrap();
let path = temp.path().join("replay-stable-boundary.dbgflow");
let id = RecordingId::from_u128(0x53);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=5 {
writer.append(&event(id, sequence)).unwrap();
}
writer.flush().unwrap();
let mut context = EventContext::new(ProcessId::new(7), ThreadId::new(9));
context.flow_id = Some(FlowId::new(1));
context.span_id = Some(SpanId::new(2));
let checkpoint = |cursor| ProjectionCheckpoint {
projection_version: CURRENT_PROJECTION_VERSION,
recording_id: id,
durable_cursor: Sequence::new(cursor),
complete: true,
active_spans: vec![ProjectedSpan {
context,
opened_sequence: Sequence::new(1),
enter_depth: 1,
is_open: true,
metadata: None,
}],
known_spans: Vec::new(),
};
writer.publish_checkpoint(&checkpoint(2)).unwrap();
let app = Server::router(ServerConfig::default(), &path).unwrap();
let first = json(
app.clone()
.oneshot(
request("/api/v1/replay?cursor=5&limit=1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(first["checkpoint"]["durable_cursor"], "2");
assert_eq!(first["deltas"][0]["sequence"], "3");
writer.publish_checkpoint(&checkpoint(4)).unwrap();
let second = json(
app.oneshot(
request("/api/v1/replay?cursor=5&boundary=2&after=3&limit=10")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(second["checkpoint"]["durable_cursor"], "2");
assert_eq!(second["deltas"][0]["sequence"], "4");
assert_eq!(second["deltas"][1]["sequence"], "5");
let lower_after = json(
Server::router(ServerConfig::default(), &path)
.unwrap()
.oneshot(
request("/api/v1/replay?cursor=5&boundary=4&after=1&limit=10")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(lower_after["checkpoint"]["durable_cursor"], "4");
assert_eq!(lower_after["deltas"][0]["sequence"], "5");
}
#[tokio::test]
async fn rolling_history_reports_exact_boundary_on_recording_pages_and_sse() {
let (_temp, path, retained) = rolling_recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
let first = retained.first_sequence.unwrap().get();
let removed = retained.removed_through_sequence.unwrap().get();
let recording = json(
app.clone()
.oneshot(request("/api/v1/recording").body(Body::empty()).unwrap())
.await
.unwrap(),
)
.await;
let boundary = &recording["durable"]["retained_prefix"];
assert_eq!(boundary["history_truncated"], true);
assert_eq!(boundary["first_cursor"], first.to_string());
assert_eq!(boundary["removed_through_cursor"], removed.to_string());
let page = json(
app.clone()
.oneshot(
request("/api/v1/events?after=0&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(page["events"][0]["sequence"], first.to_string());
assert_eq!(page["retained_prefix"]["history_truncated"], true);
assert_eq!(page["retained_prefix"]["first_cursor"], first.to_string());
assert_eq!(
page["retained_prefix"]["removed_through_cursor"],
removed.to_string()
);
let response = app
.clone()
.oneshot(
request(&format!("/api/v1/live?after={removed}"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let bytes = tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("complete rolling stream did not finish")
.unwrap()
.to_bytes();
let text = String::from_utf8(bytes.to_vec()).unwrap();
let notice = text.find("event: dbgflow.retention").unwrap();
let first_event = text.find(&format!("id: {first}")).unwrap();
assert!(notice < first_event, "{text}");
let notice_block = text[notice..first_event].to_owned();
assert!(!notice_block.lines().any(|line| line.starts_with("id: ")));
assert!(notice_block.contains(&format!("\"first_cursor\":\"{first}\"")));
assert!(notice_block.contains(&format!("\"removed_through_cursor\":\"{removed}\"")));
assert_eq!(sse_ids(&text).first().copied(), Some(first));
assert!(!text.contains("event: dbgflow.gap"), "{text}");
let header_resume = app
.clone()
.oneshot(
request(&format!("/api/v1/live?after={first}"))
.header(LAST_EVENT_ID, removed.to_string())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let header_resume =
tokio::time::timeout(Duration::from_secs(2), header_resume.into_body().collect())
.await
.expect("Last-Event-ID rolling stream did not finish")
.unwrap()
.to_bytes();
let header_resume = String::from_utf8(header_resume.to_vec()).unwrap();
assert!(header_resume.contains("event: dbgflow.retention"));
assert_eq!(sse_ids(&header_resume).first().copied(), Some(first));
let current = app
.oneshot(
request(&format!("/api/v1/live?after={first}"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let current = tokio::time::timeout(Duration::from_secs(2), current.into_body().collect())
.await
.expect("current rolling stream did not finish")
.unwrap()
.to_bytes();
let current = String::from_utf8(current.to_vec()).unwrap();
assert!(!current.contains("dbgflow.retention"), "{current}");
}
#[tokio::test]
async fn interrupted_recording_is_reported_and_does_not_reconnect_forever() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("interrupted.dbgflow");
let id = RecordingId::from_u128(0x43);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
writer.append(&event(id, 1)).unwrap();
writer.flush().unwrap();
drop(writer);
let app = Server::router(ServerConfig::default(), &path).unwrap();
let body = json(
app.clone()
.oneshot(request("/api/v1/recording").body(Body::empty()).unwrap())
.await
.unwrap(),
)
.await;
assert_eq!(body["state"], "recording");
assert_eq!(body["activity"], "interrupted");
let response = app
.oneshot(request("/api/v1/live?after=1").body(Body::empty()).unwrap())
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("interrupted recording kept SSE open")
.unwrap();
}
#[tokio::test]
async fn rejects_untrusted_hosts_and_sets_security_headers() {
let (_temp, path) = recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
for host in ["localhost:0", "LOCALHOST:0", "127.0.0.1:0", "[::1]:0"] {
let response = app
.clone()
.oneshot(
Request::get("/api/v1/capabilities")
.header(HOST, host)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK, "host {host}");
assert_eq!(response.headers()["x-content-type-options"], "nosniff");
assert_eq!(response.headers()["x-frame-options"], "DENY");
assert!(
response.headers()["content-security-policy"]
.to_str()
.unwrap()
.contains("frame-ancestors 'none'")
);
}
for host in [
None,
Some("localhost"),
Some("localhost:1"),
Some("localhost.evil.invalid:0"),
Some("127.0.0.1.evil.invalid:0"),
Some("0x7f000001:0"),
Some("[::1].evil.invalid:0"),
] {
let mut builder = Request::get("/api/v1/capabilities");
if let Some(host) = host {
builder = builder.header(HOST, host);
}
let response = app
.clone()
.oneshot(builder.body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::MISDIRECTED_REQUEST);
assert_eq!(response.headers()["x-content-type-options"], "nosniff");
}
let mut duplicate = request("/api/v1/capabilities").body(Body::empty()).unwrap();
duplicate
.headers_mut()
.append(HOST, HeaderValue::from_static("127.0.0.1:0"));
assert_eq!(
app.oneshot(duplicate).await.unwrap().status(),
StatusCode::MISDIRECTED_REQUEST
);
}
#[cfg(feature = "ui")]
#[tokio::test]
async fn optional_ui_is_served_from_the_same_origin_as_the_live_api() {
let (_temp, path) = recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
let response = app
.clone()
.oneshot(request("/").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers()["content-type"],
"text/html; charset=utf-8"
);
let html = String::from_utf8(
response
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(html.contains("/app.js"));
let response = app
.oneshot(request("/app.js").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers()["content-type"],
"application/javascript; charset=utf-8"
);
}
fn external_ui_fixture() -> TempDir {
let temp = TempDir::new().unwrap();
std::fs::write(temp.path().join("index.html"), "external index").unwrap();
std::fs::write(temp.path().join("app.js"), "external js").unwrap();
std::fs::write(temp.path().join("app.css"), "external css").unwrap();
std::fs::write(temp.path().join("globals.css"), "external globals").unwrap();
temp
}
#[tokio::test]
async fn external_ui_is_loaded_from_a_fixed_allowlist_with_correct_types() {
let (_recording_temp, path) = recording_fixture();
let ui = external_ui_fixture();
std::fs::write(ui.path().join("secret.txt"), "must not be served").unwrap();
let app = Server::router(
ServerConfig {
ui_dir: Some(ui.path().to_path_buf()),
..ServerConfig::default()
},
path,
)
.unwrap();
for (route, expected_type, expected_body) in [
("/", "text/html; charset=utf-8", "external index"),
(
"/app.js",
"application/javascript; charset=utf-8",
"external js",
),
("/app.css", "text/css; charset=utf-8", "external css"),
(
"/globals.css",
"text/css; charset=utf-8",
"external globals",
),
] {
let response = app
.clone()
.oneshot(request(route).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()["content-type"], expected_type);
assert_eq!(response.headers()["x-content-type-options"], "nosniff");
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(body.as_ref(), expected_body.as_bytes());
}
let response = app
.oneshot(request("/secret.txt").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[test]
fn external_ui_requires_every_allowlisted_asset_at_startup() {
let (_recording_temp, path) = recording_fixture();
let ui = external_ui_fixture();
std::fs::remove_file(ui.path().join("app.css")).unwrap();
let error = Server::router(
ServerConfig {
ui_dir: Some(ui.path().to_path_buf()),
..ServerConfig::default()
},
path,
)
.unwrap_err();
let message = error.to_string();
assert!(message.contains("required UI asset"));
assert!(message.contains("app.css"));
}
#[cfg(feature = "ui")]
#[tokio::test]
async fn explicit_ui_directory_takes_precedence_over_embedded_assets() {
let (_recording_temp, path) = recording_fixture();
let ui = external_ui_fixture();
let app = Server::router(
ServerConfig {
ui_dir: Some(ui.path().to_path_buf()),
..ServerConfig::default()
},
path,
)
.unwrap();
let response = app
.oneshot(request("/").body(Body::empty()).unwrap())
.await
.unwrap();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(body.as_ref(), b"external index");
}
#[cfg(not(feature = "ui"))]
#[tokio::test]
async fn server_without_ui_feature_is_api_only_until_a_directory_is_supplied() {
let (_recording_temp, path) = recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
let ui = app
.clone()
.oneshot(request("/").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(ui.status(), StatusCode::NOT_FOUND);
let api = app
.oneshot(request("/api/v1/capabilities").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(api.status(), StatusCode::OK);
}
#[tokio::test]
async fn paginates_with_exclusive_cursor_and_bounded_limit() {
let (_temp, path) = recording_fixture();
let config = ServerConfig {
max_page_limit: 3,
default_page_limit: 2,
live_batch_limit: 3,
..ServerConfig::default()
};
let app = Server::router(config, path).unwrap();
let first = app
.clone()
.oneshot(
request("/api/v1/events?limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let first = json(first).await;
assert_eq!(first["events"][0]["sequence"], "1");
assert_eq!(first["events"][1]["sequence"], "2");
assert_eq!(first["next_cursor"], "2");
assert_eq!(first["last_durable_cursor"], "5");
assert_eq!(first["has_more"], true);
assert_eq!(first["previous_cursor"], Value::Null);
let second = app
.clone()
.oneshot(
request("/api/v1/events?after=2&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let second = json(second).await;
assert_eq!(second["events"][0]["sequence"], "3");
assert_eq!(second["events"][1]["sequence"], "4");
for uri in ["/api/v1/events?limit=0", "/api/v1/events?limit=4"] {
let response = app
.clone()
.oneshot(request(uri).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
}
#[tokio::test]
async fn paginates_backward_in_ascending_order_with_directional_cursor() {
let (_temp, path) = recording_fixture();
let config = ServerConfig {
max_page_limit: 3,
default_page_limit: 2,
live_batch_limit: 3,
..ServerConfig::default()
};
let app = Server::router(config, path).unwrap();
let page = json(
app.clone()
.oneshot(
request("/api/v1/events?before=5&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(page["events"][0]["sequence"], "3");
assert_eq!(page["events"][1]["sequence"], "4");
assert_eq!(page["previous_cursor"], "3");
assert_eq!(page["next_cursor"], Value::Null);
assert_eq!(page["has_more"], true);
let oldest = json(
app.clone()
.oneshot(
request("/api/v1/events?before=3&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert_eq!(oldest["events"][0]["sequence"], "1");
assert_eq!(oldest["events"][1]["sequence"], "2");
assert_eq!(oldest["previous_cursor"], "1");
assert_eq!(oldest["has_more"], false);
let empty = json(
app.clone()
.oneshot(
request("/api/v1/events?before=1&limit=2")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap(),
)
.await;
assert!(empty["events"].as_array().unwrap().is_empty());
assert_eq!(empty["previous_cursor"], Value::Null);
assert_eq!(empty["has_more"], false);
let invalid = app
.oneshot(
request("/api/v1/events?after=1&before=5")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(invalid.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn history_pages_are_bounded_before_base64_materialization() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("page-bytes.dbgflow");
let id = RecordingId::from_u128(0x43);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=3 {
let mut envelope = event(id, sequence);
envelope.payload = Some(Payload::new(PayloadEncoding::Opaque, vec![0xaa; 100]));
writer.append(&envelope).unwrap();
}
writer.finish().unwrap();
let config = ServerConfig {
default_page_limit: 3,
max_page_limit: 3,
live_batch_limit: 3,
max_page_bytes: 1_300,
..ServerConfig::default()
};
let app = Server::router(config, &path).unwrap();
let response = app
.clone()
.oneshot(
request("/api/v1/events?limit=3")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let bytes = response.into_body().collect().await.unwrap().to_bytes();
assert!(
bytes.len() <= 1_300,
"serialized page was {} bytes",
bytes.len()
);
let body: Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(body["events"].as_array().unwrap().len(), 2);
assert_eq!(body["next_cursor"], "2");
assert_eq!(body["has_more"], true);
let backward = app
.oneshot(
request("/api/v1/events?before=4&limit=3")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(backward.status(), StatusCode::OK);
let backward = json(backward).await;
assert_eq!(backward["events"].as_array().unwrap().len(), 2);
assert_eq!(backward["events"][0]["sequence"], "2");
assert_eq!(backward["events"][1]["sequence"], "3");
assert_eq!(backward["previous_cursor"], "2");
assert_eq!(backward["has_more"], true);
let mut oversized = event(id, 4);
oversized.payload = Some(Payload::new(PayloadEncoding::Opaque, vec![0xbb; 600]));
let second_path = temp.path().join("oversized.dbgflow");
let mut second =
RecordingWriter::create(&second_path, id, RecordingOptions::default()).unwrap();
second.append(&oversized).unwrap();
second.finish().unwrap();
let too_small = ServerConfig {
max_page_bytes: MIN_PAGE_BYTES,
..ServerConfig::default()
};
let oversized_app = Server::router(too_small, second_path).unwrap();
for uri in ["/api/v1/events?limit=1", "/api/v1/events?before=5&limit=1"] {
let response = oversized_app
.clone()
.oneshot(request(uri).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
}
}
#[test]
fn page_byte_budget_has_a_hard_configuration_limit() {
for max_page_bytes in [MIN_PAGE_BYTES - 1, HARD_MAX_PAGE_BYTES + 1] {
let config = ServerConfig {
max_page_bytes,
..ServerConfig::default()
};
assert!(matches!(
config.validate(),
Err(ServerError::InvalidConfig(_))
));
}
for live_buffer_bytes in [MIN_PAGE_BYTES - 1, HARD_MAX_PAGE_BYTES + 1] {
let config = ServerConfig {
live_buffer_bytes,
..ServerConfig::default()
};
assert!(matches!(
config.validate(),
Err(ServerError::InvalidConfig(_))
));
}
}
#[test]
fn retention_notice_boundary_handles_zero_and_empty_retained_prefixes() {
let removed_zero = RetainedPrefix {
generation: 1,
first_chunk_number: 1,
first_sequence: Some(Sequence::new(1)),
removed_through_sequence: Some(Sequence::new(0)),
history_truncated: true,
};
assert!(retention_notice_required(None, removed_zero));
assert!(retention_notice_required(Some(0), removed_zero));
assert!(!retention_notice_required(Some(1), removed_zero));
let empty = RetainedPrefix {
generation: 2,
first_chunk_number: 7,
first_sequence: None,
removed_through_sequence: None,
history_truncated: true,
};
assert!(retention_notice_required(None, empty));
assert!(retention_notice_required(Some(u64::MAX), empty));
let mut cursor = Some(0);
advance_cursor_to_retained_boundary(
&mut cursor,
RetainedPrefix {
first_sequence: Some(Sequence::new(10)),
..empty
},
);
assert_eq!(cursor, Some(9));
}
#[test]
fn response_weight_is_a_conservative_bound_before_base64_encoding() {
for payload_len in [0, 1, 2, 3, 100, 1024] {
let id = RecordingId::from_u128(u128::MAX);
let mut envelope = EventEnvelope::new(
id,
Sequence::new(u64::MAX),
Timestamp::new(u64::MAX, Some(i64::MIN)),
EventContext::new(ProcessId::new(u32::MAX), ThreadId::new(u64::MAX)),
EventKind::Custom(u32::MAX),
);
envelope.context.flow_id = Some(FlowId::new(u64::MAX));
envelope.context.span_id = Some(SpanId::new(u64::MAX));
envelope.context.parent_span_id = Some(SpanId::new(u64::MAX));
envelope.payload = Some(Payload {
encoding: PayloadEncoding::Custom(u32::MAX),
flags: PayloadFlags::from_bits_retain(u32::MAX),
data: vec![0xff; payload_len],
});
let serialized = serde_json::to_vec(&EventResponse::from(&envelope)).unwrap();
assert!(
serialized.len() <= event_response_size_bound(&envelope),
"{} byte response exceeded estimate for {payload_len} payload bytes",
serialized.len()
);
}
}
#[test]
fn fanout_read_retains_only_one_payload_per_client() {
let (shutdown, _) = watch::channel(false);
let config = ServerConfig {
live_buffer_events: 4,
live_buffer_bytes: 8 * 1024,
..ServerConfig::default()
};
let hub = LiveHub::new(Arc::new(PathBuf::new()), config, shutdown);
hub.initialize(None, false);
let id = RecordingId::from_u128(0x5c);
let events = (1..=4)
.map(|sequence| {
let mut envelope = event(id, sequence);
envelope.kind = EventKind::Checkpoint;
envelope.payload = Some(Payload::new(PayloadEncoding::Opaque, vec![0xaa; 600]));
envelope
})
.collect();
hub.publish(events).unwrap();
let retained = match hub.read_after(None) {
LiveRead::Event(event) => event,
_ => panic!("fanout did not return its first event"),
};
let state = hub.state.lock().unwrap();
assert_eq!(retained.sequence.get(), 1);
assert_eq!(Arc::strong_count(&state.events[0]), 2);
assert!(
state
.events
.iter()
.skip(1)
.all(|event| Arc::strong_count(event) == 1),
"reading one event cloned payload Arcs for the rest of the ring"
);
}
#[tokio::test]
async fn sse_resumes_after_query_or_last_event_id_cursor() {
let (_temp, path) = recording_fixture();
let app = Server::router(ServerConfig::default(), path).unwrap();
let response = app
.clone()
.oneshot(request("/api/v1/live?after=3").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()[LAST_DURABLE_CURSOR], "5");
let text = String::from_utf8(
response
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(text.contains("event: dbgflow.gap\nid: 4\n"), "{text}");
assert!(text.contains("event: dbgflow.event\nid: 5\n"), "{text}");
assert!(!text.contains("id: 3\n"), "{text}");
let response = app
.clone()
.oneshot(
request("/api/v1/live")
.header(LAST_EVENT_ID, "4")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let text = String::from_utf8(
response
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(text.contains("id: 5\n"), "{text}");
assert!(!text.contains("id: 4\n"), "{text}");
let response = app
.oneshot(
request("/api/v1/live?after=1")
.header(LAST_EVENT_ID, "4")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let text = String::from_utf8(
response
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(text.contains("id: 5\n"), "{text}");
assert!(!text.contains("id: 2\n"), "{text}");
}
#[tokio::test]
async fn sse_follows_chunks_committed_after_connection() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("active.dbgflow");
let id = RecordingId::from_u128(0x55);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_poll_interval: Duration::from_millis(10),
keep_alive_interval: Duration::from_secs(1),
..ServerConfig::default()
};
let app = Server::router(config, path).unwrap();
let response = app
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap())
.await
.unwrap();
let collect = tokio::spawn(async move { response.into_body().collect().await });
tokio::time::sleep(Duration::from_millis(20)).await;
writer.append(&event(id, 1)).unwrap();
writer.flush().unwrap();
tokio::time::sleep(Duration::from_millis(20)).await;
writer.finish().unwrap();
let bytes = collect.await.unwrap().unwrap().to_bytes();
let text = String::from_utf8(bytes.to_vec()).unwrap();
assert!(text.contains("id: 1\n"), "{text}");
}
#[tokio::test]
async fn many_clients_share_exactly_one_storage_tailer() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("shared-tailer.dbgflow");
let id = RecordingId::from_u128(0x57);
let writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_poll_interval: Duration::from_millis(10),
keep_alive_interval: Duration::from_secs(1),
..ServerConfig::default()
};
let (app, diagnostics) = Server::router_with_diagnostics(config, &path).unwrap();
let mut responses = Vec::new();
for _ in 0..6 {
responses.push(
app.clone()
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap())
.await
.unwrap(),
);
}
wait_until(
|| diagnostics.completed_catchups() == 6,
"clients never completed history catch-up",
)
.await;
assert_eq!(diagnostics.tailer_starts(), 1);
let before = diagnostics.storage_polls();
tokio::time::sleep(Duration::from_millis(80)).await;
let polls = diagnostics.storage_polls() - before;
assert!(
polls <= 12,
"six clients multiplied storage polling: {polls}"
);
writer.finish().unwrap();
for response in responses {
tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("completed recording did not end SSE")
.unwrap();
}
wait_until(
|| !diagnostics.tailer_active(),
"completed shared tailer remained active",
)
.await;
}
#[tokio::test]
async fn repeated_empty_shared_polls_do_not_rescan_storage() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("stateful-empty-tail.dbgflow");
let id = RecordingId::from_u128(0x5c);
let writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_poll_interval: Duration::from_millis(5),
keep_alive_interval: Duration::from_secs(1),
..ServerConfig::default()
};
let (app, diagnostics) = Server::router_with_diagnostics(config, &path).unwrap();
let response = app
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap())
.await
.unwrap();
wait_until(
|| diagnostics.storage_polls() >= 8,
"shared tailer did not perform repeated empty polls",
)
.await;
assert_eq!(diagnostics.tail_directory_scans(), 1);
assert_eq!(diagnostics.tail_reinitializations(), 0);
assert_eq!(diagnostics.tail_frames_decoded(), 0);
writer.finish().unwrap();
tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("completed empty recording did not end SSE")
.unwrap();
}
#[tokio::test]
async fn shared_tailer_baselines_without_replaying_large_history() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("large-baseline.dbgflow");
let id = RecordingId::from_u128(0x5b);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
for sequence in 1..=10_000 {
writer.append(&event(id, sequence)).unwrap();
}
writer.finish().unwrap();
let (app, diagnostics) =
Server::router_with_diagnostics(ServerConfig::default(), path).unwrap();
let response = app
.oneshot(
request("/api/v1/live?after=9000")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("completed baseline stream did not finish")
.unwrap()
.to_bytes();
let ids = sse_ids(std::str::from_utf8(&body).unwrap());
assert_eq!(ids.len(), 1_000);
assert_eq!(ids.first(), Some(&9_001));
assert_eq!(ids.last(), Some(&10_000));
assert_eq!(diagnostics.tailer_starts(), 1);
assert_eq!(diagnostics.storage_polls(), 1);
assert_eq!(diagnostics.published_events(), 0);
assert!(
diagnostics.catchup_storage_reads() <= 16,
"1000 events caused {} catch-up scans",
diagnostics.catchup_storage_reads()
);
}
#[tokio::test]
async fn lagged_client_closes_and_recovers_from_last_event_id() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("lagged.dbgflow");
let id = RecordingId::from_u128(0x58);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_buffer_events: 2,
live_buffer_bytes: MIN_PAGE_BYTES,
live_batch_limit: 8,
live_poll_interval: Duration::from_millis(5),
keep_alive_interval: Duration::from_secs(1),
..ServerConfig::default()
};
let (app, diagnostics) = Server::router_with_diagnostics(config, &path).unwrap();
let response = app
.clone()
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap())
.await
.unwrap();
wait_until(
|| diagnostics.completed_catchups() == 1,
"client never reached shared stream",
)
.await;
for sequence in 1..=5 {
let mut envelope = event(id, sequence);
envelope.kind = EventKind::Checkpoint;
writer.append(&envelope).unwrap();
writer.flush().unwrap();
wait_until(
|| diagnostics.published_events() >= sequence,
"shared tailer did not publish durable event",
)
.await;
}
writer.finish().unwrap();
let first = tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("lagged stream did not terminate")
.unwrap()
.to_bytes();
let first = String::from_utf8(first.to_vec()).unwrap();
let first_ids = sse_ids(&first);
assert!(!first_ids.is_empty());
assert!(first_ids.len() < 5, "slow client was not treated as lagged");
assert!(!first.contains("event: dbgflow.gap"), "{first}");
let last = *first_ids.last().unwrap();
let resumed = app
.oneshot(
request("/api/v1/live")
.header(LAST_EVENT_ID, last.to_string())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let resumed = tokio::time::timeout(Duration::from_secs(2), resumed.into_body().collect())
.await
.expect("resumed stream did not finish")
.unwrap()
.to_bytes();
let resumed = String::from_utf8(resumed.to_vec()).unwrap();
assert_eq!(sse_ids(&resumed), ((last + 1)..=5).collect::<Vec<_>>());
}
#[tokio::test]
async fn subscribe_before_catchup_deduplicates_connect_race() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("connect-race.dbgflow");
let id = RecordingId::from_u128(0x59);
let mut writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_poll_interval: Duration::from_millis(5),
keep_alive_interval: Duration::from_secs(1),
..ServerConfig::default()
};
let (app, diagnostics) = Server::router_with_diagnostics(config, &path).unwrap();
let connect = tokio::spawn(
app.clone()
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap()),
);
wait_until(
|| diagnostics.subscribers() == 1,
"handler did not subscribe before catch-up",
)
.await;
for sequence in 1..=20 {
let mut envelope = event(id, sequence);
envelope.kind = EventKind::Checkpoint;
writer.append(&envelope).unwrap();
}
writer.flush().unwrap();
writer.finish().unwrap();
let response = connect.await.unwrap().unwrap();
let bytes = tokio::time::timeout(Duration::from_secs(2), response.into_body().collect())
.await
.expect("race stream did not finish")
.unwrap()
.to_bytes();
let text = String::from_utf8(bytes.to_vec()).unwrap();
assert_eq!(sse_ids(&text), (1..=20).collect::<Vec<_>>(), "{text}");
}
#[tokio::test]
async fn dropping_standalone_router_and_stream_stops_shared_tailer() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("router-drop.dbgflow");
let id = RecordingId::from_u128(0x5a);
let _writer = RecordingWriter::create(&path, id, RecordingOptions::default()).unwrap();
let config = ServerConfig {
live_poll_interval: Duration::from_millis(5),
..ServerConfig::default()
};
let (app, diagnostics) = Server::router_with_diagnostics(config, path).unwrap();
let response = app
.clone()
.oneshot(request("/api/v1/live").body(Body::empty()).unwrap())
.await
.unwrap();
wait_until(
|| diagnostics.tailer_active() && diagnostics.completed_catchups() == 1,
"shared tailer did not start",
)
.await;
drop(response);
drop(app);
wait_until(
|| !diagnostics.tailer_active(),
"standalone router leaked its shared tailer task",
)
.await;
}
#[tokio::test]
async fn runtime_periodic_flush_becomes_visible_without_manual_flush() {
let temp = TempDir::new().unwrap();
let path = temp.path().join("runtime-live.dbgflow");
let id = RecordingId::from_u128(0x66);
let mut recorder = Recorder::start(
&path,
id,
RecordingOptions::default(),
RecorderConfig {
flush_interval: Some(Duration::from_millis(10)),
..RecorderConfig::default()
},
)
.unwrap();
let app = Server::router(ServerConfig::default(), path).unwrap();
recorder.sink().try_record(event(id, 1)).unwrap();
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
loop {
let response = app
.clone()
.oneshot(
request("/api/v1/events?limit=1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = json(response).await;
if body["events"]
.as_array()
.is_some_and(|events| !events.is_empty())
{
assert_eq!(body["events"][0]["sequence"], "1");
let recording = app
.clone()
.oneshot(request("/api/v1/recording").body(Body::empty()).unwrap())
.await
.unwrap();
let recording = json(recording).await;
assert_eq!(recording["durable"]["immutable_chunk_count"], 0);
assert_eq!(recording["durable"]["has_active_prefix"], true);
assert_eq!(recording["durable"]["event_count"], 1);
assert_eq!(recording["durable"]["last_cursor"], "1");
let backward = app
.clone()
.oneshot(
request("/api/v1/events?before=2&limit=1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let backward = json(backward).await;
assert_eq!(backward["events"][0]["sequence"], "1");
assert_eq!(backward["previous_cursor"], "1");
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"periodic flush was not visible through HTTP"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
recorder.shutdown().unwrap();
}
#[tokio::test]
async fn binds_only_loopback_and_shuts_down_gracefully() {
let (_temp, path) = recording_fixture();
let invalid = ServerConfig {
bind_addr: "0.0.0.0:0".parse().unwrap(),
..ServerConfig::default()
};
assert!(matches!(
Server::bind(invalid, &path).await,
Err(ServerError::InvalidConfig(_))
));
let server = Server::bind(ServerConfig::default(), path).await.unwrap();
assert!(server.local_addr().ip().is_loopback());
server.spawn().shutdown().await.unwrap();
}
#[tokio::test]
async fn dropping_handle_aborts_server_and_releases_listener() {
let (_temp, path) = recording_fixture();
let server = Server::bind(ServerConfig::default(), path).await.unwrap();
let address = server.local_addr();
let handle = server.spawn();
drop(handle);
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
loop {
match TcpListener::bind(address).await {
Ok(listener) => {
drop(listener);
break;
}
Err(_) if tokio::time::Instant::now() < deadline => tokio::task::yield_now().await,
Err(error) => panic!("dropped server retained {address}: {error}"),
}
}
}