{
"benchmark": "codex_real_streaming",
"benchmark_version": "1",
"date": "2026-05-24",
"endpoint": "https://chatgpt.com/backend-api/codex/responses",
"model": "gpt-5.4-mini",
"accept_encoding": "identity",
"warmup_count": 1,
"sample_count": 10,
"inter_request_delay_ms": 2000,
"lead_alternation": "SR/RS/SR/RS/SR",
"primary_claim": "pass",
"primary_claim_passed": 10,
"primary_claim_total": 10,
"primary_claim_threshold": 8,
"primary_claim_definition": "status_code==200 AND delta_count>=1 AND response.completed received within 30s; pass if >=ceil(0.8*N) of N samples meet condition (counted per-pair, both clients must pass for the pair to count)",
"failures": [],
"environment": {
"os": "macos",
"arch": "aarch64",
"specter_version": "3.2.0"
},
"protocol_mismatch": false,
"rows": [
{
"client": "specter",
"warmup": true,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 559.925708,
"total_wall_time_ms": 660.4339580000001,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 477.57273656640086,
"pool_reuse_delta_specter": 0,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656815493
},
{
"client": "reqwest",
"warmup": true,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 727.4043340000001,
"total_wall_time_ms": 824.811292,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 492.7779389230084,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656818400
},
{
"client": "specter",
"warmup": false,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 558.7953329999999,
"total_wall_time_ms": 670.698083,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 428.94388207617743,
"pool_reuse_delta_specter": 1,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656821172
},
{
"client": "reqwest",
"warmup": false,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 578.9905,
"total_wall_time_ms": 740.687042,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 296.8523594029611,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656823985
},
{
"client": "reqwest",
"warmup": false,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 930.1407499999999,
"total_wall_time_ms": 1040.36725,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 435.466970283915,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656827026
},
{
"client": "specter",
"warmup": false,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 625.135875,
"total_wall_time_ms": 717.7531670000001,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 518.2617518119617,
"pool_reuse_delta_specter": 1,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656829824
},
{
"client": "specter",
"warmup": false,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 551.5627920000001,
"total_wall_time_ms": 643.1352919999999,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 524.1748341478071,
"pool_reuse_delta_specter": 1,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656832541
},
{
"client": "reqwest",
"warmup": false,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 924.393291,
"total_wall_time_ms": 968.917083,
"total_chars": 66,
"delta_count": 19,
"completed": true,
"chars_per_sec": 1482.3535246054491,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656835570
},
{
"client": "reqwest",
"warmup": false,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 1041.348667,
"total_wall_time_ms": 1087.5385419999998,
"total_chars": 66,
"delta_count": 19,
"completed": true,
"chars_per_sec": 1428.8845769771033,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656838718
},
{
"client": "specter",
"warmup": false,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 597.064084,
"total_wall_time_ms": 708.60225,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 430.34596785462634,
"pool_reuse_delta_specter": 1,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656841494
},
{
"client": "specter",
"warmup": false,
"sample_index": 8,
"pair_index": 4,
"lead_in_pair": true,
"status": "ok",
"status_code": 200,
"ttfb_ms": 531.023833,
"total_wall_time_ms": 636.88075,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 453.4422630124394,
"pool_reuse_delta_specter": 1,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656844230
},
{
"client": "reqwest",
"warmup": false,
"sample_index": 9,
"pair_index": 4,
"lead_in_pair": false,
"status": "ok",
"status_code": 200,
"ttfb_ms": 560.4407500000001,
"total_wall_time_ms": 717.6911249999999,
"total_chars": 48,
"delta_count": 19,
"completed": true,
"chars_per_sec": 305.2456949625719,
"pool_reuse_delta_specter": null,
"protocol_used": "HTTP/2",
"epoch_ms": 1779656846980
}
],
"summary": {
"specter": {
"median_ttfb_ms": 558.7953329999999,
"p95_ttfb_ms": 619.5215168000001,
"median_total_wall_time_ms": 670.698083,
"p95_total_wall_time_ms": 715.9229836000001,
"median_chars_per_sec": 453.4422630124394,
"passing_samples": 5
},
"reqwest": {
"median_ttfb_ms": 924.393291,
"p95_ttfb_ms": 1019.1070835999999,
"median_total_wall_time_ms": 968.917083,
"p95_total_wall_time_ms": 1078.1042835999997,
"median_chars_per_sec": 435.466970283915,
"passing_samples": 5
},
"protocol_used_specter": "HTTP/2",
"protocol_used_reqwest": "HTTP/2",
"protocol_mismatch": false,
"comparison": {
"ttfb_difference_ms": -234.34640819999996,
"ttfb_ci_95": [
-479.59571780974284,
10.902901409742924
],
"ttfb_ci_covers_zero": true,
"wall_time_difference_ms": -235.6263,
"wall_time_ci_95": [
-419.3844373685183,
-51.868162631481624
],
"wall_time_ci_covers_zero": false,
"ttfb_wilcoxon_p_value": 0.02952904975182835,
"wall_time_wilcoxon_p_value": 0.02952904975182835,
"interpretation": "Mixed: TTFB CI=[-479.6, 10.9], wall CI=[-419.4, -51.9]"
}
}
}