{
"benchmark": "codex_ws_streaming",
"benchmark_version": "1",
"date": "2026-05-24",
"endpoint": "wss://chatgpt.com/backend-api/codex/responses",
"model": "gpt-5.4-mini",
"warmup_count": 1,
"sample_count": 10,
"inter_request_delay_ms": 2000,
"lead_alternation": "SR/RS/SR/RS/SR",
"primary_claim": "pass",
"primary_claim_passed": 10,
"primary_claim_total": 10,
"primary_claim_threshold": 8,
"primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
"failures": [],
"environment": {
"os": "macos",
"arch": "aarch64",
"specter_version": "3.2.0",
"tokio_tungstenite_version": "0.24"
},
"rows": [
{
"client": "specter",
"warmup": true,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 276.826417,
"ttfb_ms": 872.6584160000001,
"total_wall_time_ms": 948.361708,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 871.8247021543009,
"epoch_ms": 1779656763426
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 326.65925,
"ttfb_ms": 870.9649579999999,
"total_wall_time_ms": 987.0450830000001,
"total_chars": 57,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 491.04013283927725,
"epoch_ms": 1779656766743
},
{
"client": "specter",
"warmup": false,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 382.296292,
"ttfb_ms": 956.997,
"total_wall_time_ms": 1001.8789589999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1470.5240473126428,
"epoch_ms": 1779656770154
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 391.57470900000004,
"ttfb_ms": 518.862666,
"total_wall_time_ms": 619.671458,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 476.1489454213476,
"epoch_ms": 1779656773202
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 330.541042,
"ttfb_ms": 988.818625,
"total_wall_time_ms": 1053.517083,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 741.9033078037197,
"epoch_ms": 1779656776698
},
{
"client": "specter",
"warmup": false,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 406.633125,
"ttfb_ms": 546.9919170000001,
"total_wall_time_ms": 637.6072499999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 529.7116769410325,
"epoch_ms": 1779656779842
},
{
"client": "specter",
"warmup": false,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 378.943166,
"ttfb_ms": 558.8975,
"total_wall_time_ms": 671.126875,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 427.69551198160013,
"epoch_ms": 1779656782989
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 382.96175,
"ttfb_ms": 545.487834,
"total_wall_time_ms": 655.3312500000001,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 436.9856815086667,
"epoch_ms": 1779656786137
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 471.421875,
"ttfb_ms": 446.629417,
"total_wall_time_ms": 542.70225,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 499.6209490356131,
"epoch_ms": 1779656789283
},
{
"client": "specter",
"warmup": false,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 392.192791,
"ttfb_ms": 796.2658329999999,
"total_wall_time_ms": 842.375,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1041.00774581332,
"epoch_ms": 1779656792539
},
{
"client": "specter",
"warmup": false,
"sample_index": 8,
"pair_index": 4,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 309.355292,
"ttfb_ms": 533.622166,
"total_wall_time_ms": 724.243208,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 251.8085070587328,
"epoch_ms": 1779656795575
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 9,
"pair_index": 4,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 373.38100000000003,
"ttfb_ms": 767.5918750000001,
"total_wall_time_ms": 767.6468749999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 66000.0,
"epoch_ms": 1779656798725
}
],
"summary": {
"specter": {
"median_ttfb_ms": 558.8975,
"p95_ttfb_ms": 924.8507665999999,
"median_total_wall_time_ms": 724.243208,
"p95_total_wall_time_ms": 969.9781671999999,
"median_handshake_ms": 382.296292,
"median_chars_per_sec": 529.7116769410325,
"passing_samples": 5
},
"tungstenite": {
"median_ttfb_ms": 545.487834,
"p95_ttfb_ms": 944.573275,
"median_total_wall_time_ms": 655.3312500000001,
"p95_total_wall_time_ms": 996.3430414,
"median_handshake_ms": 382.96175,
"median_chars_per_sec": 499.6209490356131,
"passing_samples": 5
},
"comparison": {
"ttfb_difference_ms": 25.076799799999982,
"ttfb_ci_95": [
-439.9324394089911,
490.0860390089911
],
"ttfb_ci_covers_zero": true,
"wall_time_difference_ms": 47.672475199999965,
"wall_time_ci_95": [
-344.7352179884089,
440.0801683884088
],
"wall_time_ci_covers_zero": true,
"handshake_difference_ms": -16.091942000000007,
"handshake_ci_95": [
-92.04999105474846,
59.866107054748454
],
"handshake_ci_covers_zero": true,
"ttfb_wilcoxon_p_value": 0.6062966923611464,
"wall_time_wilcoxon_p_value": 0.6062966923611464,
"interpretation": "All differences within network noise at n=5. Both clients streamed successfully from Codex WS."
}
}
}