{
"benchmark": "codex_ws_streaming",
"benchmark_version": "1",
"date": "2026-05-24",
"endpoint": "wss://chatgpt.com/backend-api/codex/responses",
"model": "gpt-5.4-mini",
"warmup_count": 4,
"sample_count": 30,
"inter_request_delay_ms": 2000,
"lead_alternation": "SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR",
"primary_claim": "pass",
"primary_claim_passed": 30,
"primary_claim_total": 30,
"primary_claim_threshold": 24,
"primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
"failures": [],
"environment": {
"os": "macos",
"arch": "aarch64",
"specter_version": "4.0.0",
"tokio_tungstenite_version": "0.24"
},
"rows": [
{
"client": "specter",
"warmup": true,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 359.980167,
"ttfb_ms": 558.5572920000001,
"total_wall_time_ms": 671.0193330000001,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 426.8106782803275,
"epoch_ms": 1779657900923
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 261.34545799999995,
"ttfb_ms": 703.967875,
"total_wall_time_ms": 704.0456670000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 66000.0,
"epoch_ms": 1779657903894
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 479.29029099999997,
"ttfb_ms": 940.354542,
"total_wall_time_ms": 983.193167,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1120.4841425232491,
"epoch_ms": 1779657907384
},
{
"client": "specter",
"warmup": true,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 330.09325,
"ttfb_ms": 476.756625,
"total_wall_time_ms": 576.4121250000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 662.2815599741103,
"epoch_ms": 1779657910335
},
{
"client": "specter",
"warmup": true,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 263.873958,
"ttfb_ms": 856.349417,
"total_wall_time_ms": 898.26675,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1574.5276542283837,
"epoch_ms": 1779657913532
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 432.60425000000004,
"ttfb_ms": 842.1440839999999,
"total_wall_time_ms": 888.8922500000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1411.8200915090385,
"epoch_ms": 1779657916890
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 342.800208,
"ttfb_ms": 551.6554580000001,
"total_wall_time_ms": 650.163042,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 487.27212718972,
"epoch_ms": 1779657919926
},
{
"client": "specter",
"warmup": true,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 482.838458,
"ttfb_ms": 1037.436291,
"total_wall_time_ms": 1088.115458,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1302.31027672574,
"epoch_ms": 1779657923501
},
{
"client": "specter",
"warmup": false,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 302.754166,
"ttfb_ms": 834.534167,
"total_wall_time_ms": 879.957,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1056.7372581098152,
"epoch_ms": 1779657926689
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 363.74925,
"ttfb_ms": 923.9611669999999,
"total_wall_time_ms": 1960.0183339999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 46.32948984754237,
"epoch_ms": 1779657931780
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 323.477958,
"ttfb_ms": 607.508333,
"total_wall_time_ms": 859.2747499999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 190.6529098358659,
"epoch_ms": 1779657934970
},
{
"client": "specter",
"warmup": false,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 277.633334,
"ttfb_ms": 996.3853750000001,
"total_wall_time_ms": 1023.348375,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 2447.798835441163,
"epoch_ms": 1779657938295
},
{
"client": "specter",
"warmup": false,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 448.446125,
"ttfb_ms": 548.812542,
"total_wall_time_ms": 640.07875,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 525.933979858131,
"epoch_ms": 1779657941424
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 312.74583299999995,
"ttfb_ms": 1045.096333,
"total_wall_time_ms": 1045.147458,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 48000.0,
"epoch_ms": 1779657944804
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 491.58275000000003,
"ttfb_ms": 962.88,
"total_wall_time_ms": 972.208666,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 7074.966560063359,
"epoch_ms": 1779657948298
},
{
"client": "specter",
"warmup": false,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 295.52858399999997,
"ttfb_ms": 1200.877792,
"total_wall_time_ms": 1209.3378750000002,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 5673.703201256896,
"epoch_ms": 1779657951867
},
{
"client": "specter",
"warmup": false,
"sample_index": 8,
"pair_index": 4,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 352.741792,
"ttfb_ms": 423.259333,
"total_wall_time_ms": 525.760416,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 643.8956357173323,
"epoch_ms": 1779657954783
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 9,
"pair_index": 4,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 339.68795800000004,
"ttfb_ms": 489.765833,
"total_wall_time_ms": 601.167958,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 430.87149369906535,
"epoch_ms": 1779657957764
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 10,
"pair_index": 5,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 282.28033400000004,
"ttfb_ms": 655.4414589999999,
"total_wall_time_ms": 705.6816249999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1313.68992690032,
"epoch_ms": 1779657960767
},
{
"client": "specter",
"warmup": false,
"sample_index": 11,
"pair_index": 5,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 364.12575000000004,
"ttfb_ms": 708.686709,
"total_wall_time_ms": 731.52725,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 2101.5264043001407,
"epoch_ms": 1779657963903
},
{
"client": "specter",
"warmup": false,
"sample_index": 12,
"pair_index": 6,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 278.847042,
"ttfb_ms": 2410.487291,
"total_wall_time_ms": 2475.858416,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1009.620073082724,
"epoch_ms": 1779657968694
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 13,
"pair_index": 6,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 301.055792,
"ttfb_ms": 839.4427089999999,
"total_wall_time_ms": 881.2378749999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1148.458173368662,
"epoch_ms": 1779657971908
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 14,
"pair_index": 7,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 277.153166,
"ttfb_ms": 501.983125,
"total_wall_time_ms": 589.198333,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 550.3627303164828,
"epoch_ms": 1779657974820
},
{
"client": "specter",
"warmup": false,
"sample_index": 15,
"pair_index": 7,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 357.637792,
"ttfb_ms": 632.876459,
"total_wall_time_ms": 632.920167,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 48000.0,
"epoch_ms": 1779657977850
},
{
"client": "specter",
"warmup": false,
"sample_index": 16,
"pair_index": 8,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 343.525875,
"ttfb_ms": 462.809458,
"total_wall_time_ms": 564.536291,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 471.8519055832594,
"epoch_ms": 1779657980790
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 17,
"pair_index": 8,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 441.631375,
"ttfb_ms": 2964.638834,
"total_wall_time_ms": 2966.832334,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 21882.835650783294,
"epoch_ms": 1779657986213
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 18,
"pair_index": 9,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 352.192666,
"ttfb_ms": 778.752208,
"total_wall_time_ms": 780.489417,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 37991.97448320826,
"epoch_ms": 1779657989365
},
{
"client": "specter",
"warmup": false,
"sample_index": 19,
"pair_index": 9,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 341.218916,
"ttfb_ms": 586.219666,
"total_wall_time_ms": 782.3876250000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 336.4463816438033,
"epoch_ms": 1779657992491
},
{
"client": "specter",
"warmup": false,
"sample_index": 20,
"pair_index": 10,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 891.098417,
"ttfb_ms": 1013.3256250000001,
"total_wall_time_ms": 1059.614208,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1425.8375548026634,
"epoch_ms": 1779657996467
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 21,
"pair_index": 10,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 319.368291,
"ttfb_ms": 550.087625,
"total_wall_time_ms": 670.284416,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 399.3451039803552,
"epoch_ms": 1779657999493
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 22,
"pair_index": 11,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 435.27824999999996,
"ttfb_ms": 542.326167,
"total_wall_time_ms": 574.082333,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 1511.5174797864504,
"epoch_ms": 1779658002543
},
{
"client": "specter",
"warmup": false,
"sample_index": 23,
"pair_index": 11,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 412.78441599999996,
"ttfb_ms": 781.89875,
"total_wall_time_ms": 832.526625,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 948.0943057554753,
"epoch_ms": 1779658005806
},
{
"client": "specter",
"warmup": false,
"sample_index": 24,
"pair_index": 12,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 300.691875,
"ttfb_ms": 637.999375,
"total_wall_time_ms": 671.6348340000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1962.2149351373453,
"epoch_ms": 1779658008813
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 25,
"pair_index": 12,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 301.407,
"ttfb_ms": 469.925292,
"total_wall_time_ms": 612.0917919999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 337.6322832734859,
"epoch_ms": 1779658011769
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 26,
"pair_index": 13,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 282.51741699999997,
"ttfb_ms": 981.366542,
"total_wall_time_ms": 999.70225,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 3599.533762208678,
"epoch_ms": 1779658015078
},
{
"client": "specter",
"warmup": false,
"sample_index": 27,
"pair_index": 13,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 328.480917,
"ttfb_ms": 429.56525,
"total_wall_time_ms": 533.536875,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 461.6644204608709,
"epoch_ms": 1779658017993
},
{
"client": "specter",
"warmup": false,
"sample_index": 28,
"pair_index": 14,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 295.916292,
"ttfb_ms": 540.18575,
"total_wall_time_ms": 623.498334,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 792.1972507778656,
"epoch_ms": 1779658020947
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 29,
"pair_index": 14,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 304.215667,
"ttfb_ms": 694.080375,
"total_wall_time_ms": 762.3283749999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 703.3173133278638,
"epoch_ms": 1779658024057
}
],
"summary": {
"specter": {
"median_ttfb_ms": 637.999375,
"p95_ttfb_ms": 1563.7606416999988,
"median_total_wall_time_ms": 731.52725,
"p95_total_wall_time_ms": 1589.2940372999988,
"median_handshake_ms": 341.218916,
"median_chars_per_sec": 1009.620073082724,
"passing_samples": 15
},
"tungstenite": {
"median_ttfb_ms": 694.080375,
"p95_ttfb_ms": 1620.959083299998,
"median_total_wall_time_ms": 780.489417,
"p95_total_wall_time_ms": 2262.062533999999,
"median_handshake_ms": 319.368291,
"median_chars_per_sec": 1148.458173368662,
"passing_samples": 15
},
"comparison": {
"ttfb_difference_ms": -53.28883066666665,
"ttfb_ci_95": [
-517.5300498466272,
410.9523885132939
],
"ttfb_ci_covers_zero": true,
"wall_time_difference_ms": -119.54819166666661,
"wall_time_ci_95": [
-587.1572054081456,
348.06082207481234
],
"wall_time_ci_covers_zero": true,
"handshake_difference_ms": 30.87250573333333,
"handshake_ci_95": [
-62.92134681689642,
124.66635828356307
],
"handshake_ci_covers_zero": true,
"ttfb_wilcoxon_p_value": 0.5787851532947169,
"wall_time_wilcoxon_p_value": 0.46605340600573286,
"interpretation": "All differences within network noise at n=15. Both clients streamed successfully from Codex WS."
}
}
}