{
"benchmark": "codex_ws_streaming",
"benchmark_version": "1",
"date": "2026-05-24",
"endpoint": "wss://chatgpt.com/backend-api/codex/responses",
"model": "gpt-5.4-mini",
"warmup_count": 4,
"sample_count": 30,
"inter_request_delay_ms": 2000,
"lead_alternation": "SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR/RS/SR",
"primary_claim": "pass",
"primary_claim_passed": 30,
"primary_claim_total": 30,
"primary_claim_threshold": 24,
"primary_claim_definition": "WebSocket connect, send response.create, receive >=1 response.output_text.delta AND response.completed text frame within 30s; pass if >=ceil(0.8*N) of N samples (counted per-pair, both clients must pass)",
"failures": [],
"environment": {
"os": "macos",
"arch": "aarch64",
"specter_version": "3.2.0",
"tokio_tungstenite_version": "0.24"
},
"rows": [
{
"client": "specter",
"warmup": true,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 347.388333,
"ttfb_ms": 511.020625,
"total_wall_time_ms": 648.6850830000001,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 348.6738748501082,
"epoch_ms": 1779657212372
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 381.394041,
"ttfb_ms": 1040.251583,
"total_wall_time_ms": 1080.761833,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1184.8853068050666,
"epoch_ms": 1779657215855
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 281.301458,
"ttfb_ms": 1987.057041,
"total_wall_time_ms": 2620.265666,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 104.231050232457,
"epoch_ms": 1779657220824
},
{
"client": "specter",
"warmup": true,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 297.438084,
"ttfb_ms": 802.348667,
"total_wall_time_ms": 837.049625,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1901.9647814910463,
"epoch_ms": 1779657223987
},
{
"client": "specter",
"warmup": true,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 296.685166,
"ttfb_ms": 781.181417,
"total_wall_time_ms": 829.6481249999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1361.7594989121212,
"epoch_ms": 1779657227125
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 270.61337499999996,
"ttfb_ms": 556.994042,
"total_wall_time_ms": 663.439375,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 450.9356929720911,
"epoch_ms": 1779657230097
},
{
"client": "tungstenite",
"warmup": true,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 293.520291,
"ttfb_ms": 3052.730458,
"total_wall_time_ms": 3097.175417,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1079.9874964447617,
"epoch_ms": 1779657235516
},
{
"client": "specter",
"warmup": true,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 374.96562500000005,
"ttfb_ms": 1057.221334,
"total_wall_time_ms": 1107.660959,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1308.495057209486,
"epoch_ms": 1779657239001
},
{
"client": "specter",
"warmup": false,
"sample_index": 0,
"pair_index": 0,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 308.534042,
"ttfb_ms": 564.869208,
"total_wall_time_ms": 678.119125,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 423.841370232527,
"epoch_ms": 1779657242031
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 1,
"pair_index": 0,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 312.112958,
"ttfb_ms": 608.430083,
"total_wall_time_ms": 656.707,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1367.1129827946545,
"epoch_ms": 1779657245035
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 2,
"pair_index": 1,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 328.427209,
"ttfb_ms": 526.2926669999999,
"total_wall_time_ms": 646.568125,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 399.08390953705594,
"epoch_ms": 1779657248039
},
{
"client": "specter",
"warmup": false,
"sample_index": 3,
"pair_index": 1,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 343.074333,
"ttfb_ms": 399.600416,
"total_wall_time_ms": 566.0376660000001,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 288.39697844082366,
"epoch_ms": 1779657250951
},
{
"client": "specter",
"warmup": false,
"sample_index": 4,
"pair_index": 2,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 308.633125,
"ttfb_ms": 848.991375,
"total_wall_time_ms": 1053.329875,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 322.99346427618883,
"epoch_ms": 1779657254315
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 5,
"pair_index": 2,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 365.376625,
"ttfb_ms": 1135.932167,
"total_wall_time_ms": 1379.374792,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 197.1717155120225,
"epoch_ms": 1779657258119
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 6,
"pair_index": 3,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 294.066,
"ttfb_ms": 873.487542,
"total_wall_time_ms": 1002.748459,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 371.3419424372487,
"epoch_ms": 1779657261483
},
{
"client": "specter",
"warmup": false,
"sample_index": 7,
"pair_index": 3,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 287.327917,
"ttfb_ms": 522.1443340000001,
"total_wall_time_ms": 636.0331249999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 421.46377688740296,
"epoch_ms": 1779657264448
},
{
"client": "specter",
"warmup": false,
"sample_index": 8,
"pair_index": 4,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 325.89833400000003,
"ttfb_ms": 487.487417,
"total_wall_time_ms": 635.052792,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 325.2795582974665,
"epoch_ms": 1779657267412
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 9,
"pair_index": 4,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 380.74449999999996,
"ttfb_ms": 582.1409170000001,
"total_wall_time_ms": 662.529167,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 597.102188441719,
"epoch_ms": 1779657270554
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 10,
"pair_index": 5,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 651.109708,
"ttfb_ms": 533.0572500000001,
"total_wall_time_ms": 634.314541,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 651.8049154603599,
"epoch_ms": 1779657273883
},
{
"client": "specter",
"warmup": false,
"sample_index": 11,
"pair_index": 5,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 345.46629099999996,
"ttfb_ms": 622.3585830000001,
"total_wall_time_ms": 622.432417,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 66000.0,
"epoch_ms": 1779657276894
},
{
"client": "specter",
"warmup": false,
"sample_index": 12,
"pair_index": 6,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 362.70383400000003,
"ttfb_ms": 538.38325,
"total_wall_time_ms": 743.19475,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 234.36184003339653,
"epoch_ms": 1779657280002
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 13,
"pair_index": 6,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 353.145375,
"ttfb_ms": 789.4452090000001,
"total_wall_time_ms": 844.241,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 1204.4720734116263,
"epoch_ms": 1779657283232
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 14,
"pair_index": 7,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 289.91654100000005,
"ttfb_ms": 626.07325,
"total_wall_time_ms": 769.825042,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 333.90888094111546,
"epoch_ms": 1779657286295
},
{
"client": "specter",
"warmup": false,
"sample_index": 15,
"pair_index": 7,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 424.33829099999997,
"ttfb_ms": 492.146417,
"total_wall_time_ms": 597.9249169999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 453.7784143280536,
"epoch_ms": 1779657289439
},
{
"client": "specter",
"warmup": false,
"sample_index": 16,
"pair_index": 8,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 391.61400000000003,
"ttfb_ms": 629.013667,
"total_wall_time_ms": 752.781959,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 533.2545107756679,
"epoch_ms": 1779657292642
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 17,
"pair_index": 8,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 336.187167,
"ttfb_ms": 909.93625,
"total_wall_time_ms": 956.388958,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1420.799837977152,
"epoch_ms": 1779657295945
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 18,
"pair_index": 9,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 271.644208,
"ttfb_ms": 541.211833,
"total_wall_time_ms": 654.938833,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 422.06336226225926,
"epoch_ms": 1779657298898
},
{
"client": "specter",
"warmup": false,
"sample_index": 19,
"pair_index": 9,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 332.317125,
"ttfb_ms": 649.957583,
"total_wall_time_ms": 790.7364580000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 468.8203396994042,
"epoch_ms": 1779657302024
},
{
"client": "specter",
"warmup": false,
"sample_index": 20,
"pair_index": 10,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 357.867292,
"ttfb_ms": 597.485625,
"total_wall_time_ms": 783.3462079999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 355.1048798765472,
"epoch_ms": 1779657305167
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 21,
"pair_index": 10,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 433.166458,
"ttfb_ms": 493.442916,
"total_wall_time_ms": 590.278,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 681.571154520814,
"epoch_ms": 1779657308311
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 22,
"pair_index": 11,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 436.42674999999997,
"ttfb_ms": 574.932833,
"total_wall_time_ms": 707.983625,
"total_chars": 66,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 496.0511621757201,
"epoch_ms": 1779657311503
},
{
"client": "specter",
"warmup": false,
"sample_index": 23,
"pair_index": 11,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 329.183625,
"ttfb_ms": 955.92325,
"total_wall_time_ms": 1005.1827499999999,
"total_chars": 48,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 974.4313279671964,
"epoch_ms": 1779657314880
},
{
"client": "specter",
"warmup": false,
"sample_index": 24,
"pair_index": 12,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 344.684416,
"ttfb_ms": 857.204125,
"total_wall_time_ms": 902.4952499999999,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1457.239139014543,
"epoch_ms": 1779657318164
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 25,
"pair_index": 12,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 281.07275,
"ttfb_ms": 644.1885000000001,
"total_wall_time_ms": 715.045875,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 931.4485612824362,
"epoch_ms": 1779657321188
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 26,
"pair_index": 13,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 326.566292,
"ttfb_ms": 680.792458,
"total_wall_time_ms": 730.597041,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1325.1792510741436,
"epoch_ms": 1779657324283
},
{
"client": "specter",
"warmup": false,
"sample_index": 27,
"pair_index": 13,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 374.977083,
"ttfb_ms": 1115.103625,
"total_wall_time_ms": 1215.503625,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 478.08764940239,
"epoch_ms": 1779657327932
},
{
"client": "specter",
"warmup": false,
"sample_index": 28,
"pair_index": 14,
"lead_in_pair": true,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 265.609875,
"ttfb_ms": 795.320958,
"total_wall_time_ms": 844.0667910000001,
"total_chars": 66,
"delta_count": 19,
"frame_count": 30,
"completed": true,
"chars_per_sec": 1353.96188634216,
"epoch_ms": 1779657331052
},
{
"client": "tungstenite",
"warmup": false,
"sample_index": 29,
"pair_index": 14,
"lead_in_pair": false,
"status": "ok",
"handshake_status": 101,
"handshake_ms": 275.741833,
"ttfb_ms": 5392.9285,
"total_wall_time_ms": 10015.015,
"total_chars": 48,
"delta_count": 19,
"frame_count": 28,
"completed": true,
"chars_per_sec": 10.38492031683094,
"epoch_ms": 1779657345074
}
],
"summary": {
"specter": {
"median_ttfb_ms": 622.3585830000001,
"p95_ttfb_ms": 1003.6773624999998,
"median_total_wall_time_ms": 752.781959,
"p95_total_wall_time_ms": 1101.9819999999997,
"median_handshake_ms": 343.074333,
"median_chars_per_sec": 453.7784143280536,
"passing_samples": 15
},
"tungstenite": {
"median_ttfb_ms": 626.07325,
"p95_ttfb_ms": 2413.0310668999955,
"median_total_wall_time_ms": 715.045875,
"p95_total_wall_time_ms": 3970.0668543999905,
"median_handshake_ms": 328.427209,
"median_chars_per_sec": 597.102188441719,
"passing_samples": 15
},
"comparison": {
"ttfb_difference_ms": -322.4201694666666,
"ttfb_ci_95": [
-990.657391059648,
345.8170521263148
],
"ttfb_ci_covers_zero": true,
"wall_time_difference_ms": -609.3545166666667,
"wall_time_ci_95": [
-1927.312404370739,
708.6033710374055
],
"wall_time_ci_covers_zero": true,
"handshake_difference_ms": -15.56498606666665,
"handshake_ci_95": [
-71.90217823229375,
40.772206098960446
],
"handshake_ci_covers_zero": true,
"ttfb_wilcoxon_p_value": 0.22161547183156394,
"wall_time_wilcoxon_p_value": 0.2947489413971336,
"interpretation": "All differences within network noise at n=15. Both clients streamed successfully from Codex WS."
}
}
}