[
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.90",
"system": "hnswlib",
"value": 9480.442447565747,
"ci": [
9417.422568871087,
9697.221801401593
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[2]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "9,480"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.90",
"system": "hms",
"value": 23293.629504996767,
"ci": [
22814.748394915856,
23733.858833120175
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[11]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "23,294"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.90",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.95",
"system": "hnswlib",
"value": 7630.414893330864,
"ci": [
7474.952830643235,
7814.989707482342
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "7,630"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.95",
"system": "hms",
"value": 17480.964142532004,
"ci": [
15253.129845587078,
17833.849020964924
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "17,481"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.95",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.99",
"system": "hnswlib",
"value": 3192.520978113118,
"ci": [
3080.7832588347464,
3298.7230563365106
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[6]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "3,193"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.99",
"system": "hms",
"value": 8794.271294386524,
"ci": [
8591.235496775094,
9082.365742451297
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[5]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "8,794"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps@recall0.99",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency@recall0.95",
"system": "hnswlib",
"value": 133.32475,
"ci": [
128.75775,
136.15449999999998
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "133"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency@recall0.95",
"system": "hms",
"value": 62.091,
"ci": [
55.067,
62.514
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "62"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency@recall0.95",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency@recall0.95",
"system": "hnswlib",
"value": 219.73541500000005,
"ci": [
216.20628,
228.29286000000002
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "220"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency@recall0.95",
"system": "hms",
"value": 104.57,
"ci": [
84.128,
110.87700000000001
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "105"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency@recall0.95",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hnswlib",
"value": 182.4948315215006,
"ci": [
182.2122630720005,
182.7773999710007
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "182.49"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hms",
"value": 312.6031632165,
"ci": [
310.021226209,
315.185100224
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[graph_built]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "312.60"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hnswlib",
"value": 1524.958984375,
"ci": [
1520.74609375,
1529.171875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,525"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": 2967.65234375,
"ci": [
2957.69140625,
2977.61328125
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[graph_built]",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2,968"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hnswlib",
"value": 1019.3671875,
"ci": [
1018.99609375,
1019.375
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].rss_steady",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,019"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hms",
"value": 2203.828125,
"ci": [
1261.109375,
2223.64453125
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].rss_steady",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2,204"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "index_bytes_mib",
"system": "hnswlib",
"value": 873.813777923584,
"ci": [
873.813777923584,
873.813777923584
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "873.8"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "index_bytes_mib",
"system": "hms",
"value": 446.3023262023926,
"ci": [
446.3023262023926,
446.3023262023926
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "446.3"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "index_bytes_mib",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hnswlib",
"value": 0.748887823,
"ci": [
0.536470403,
0.77115552
],
"n": 3,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.75"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hnswlib",
"value": 6.010148169989407,
"ci": [
6.010148169989407,
6.010148169989407
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "6.010"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hnswlib",
"value": 3.3136410399856686,
"ci": [
3.3136410399856686,
3.3136410399856686
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "3.314"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/m"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hnswlib",
"value": 61.750614166259766,
"ci": [
61.750614166259766,
61.750614166259766
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "61.8"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hms",
"value": 3.378631591796875,
"ci": [
3.378631591796875,
3.378631591796875
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "3.4"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hnswlib",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)",
"file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "hnswlib",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.26537791112001474,
"ci": [
0.23702131021485068,
0.2947007216218275
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2654"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.26781574230549227,
"ci": [
0.2371816120958971,
0.2968809113532711
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2678"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.2656768159122737,
"ci": [
0.23666905387824008,
0.2935669445083154
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2657"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.26427698472308014,
"ci": [
0.23619095893739991,
0.29259476476656954
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2643"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "hms",
"value": 0.5422345043845044,
"ci": [
0.5079547619047619,
0.5755660173160173
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.5422"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.5545597791097792,
"ci": [
0.5197631368631369,
0.5880408036408037
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.5546"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "lucene",
"value": 0.5444845043845044,
"ci": [
0.5103370795870796,
0.5756238816738817
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.5445"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.5405661283161284,
"ci": [
0.5067542235542235,
0.5743550172050172
],
"n": 500,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.5406"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "hms",
"value": 3611.9250624371816,
"ci": [
3305.9404867767275,
3636.4703502390053
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "3,612"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 26.391160458321984,
"ci": [
25.512147136220875,
26.854216080674412
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "26"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "lucene",
"value": 1267.696321226662,
"ci": [
1136.4776689832,
1286.2379536098217
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,268"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "tantivy",
"value": 1617.0952478997635,
"ci": [
1470.224415289986,
1651.386284785746
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,617"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "hms",
"value": 269.099,
"ci": [
266.955,
293.864
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "269"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 34691.752,
"ci": [
34254.818,
35827.793
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "34,692"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "lucene",
"value": 721.343,
"ci": [
713.381,
802.49
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "721"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "tantivy",
"value": 600.18,
"ci": [
598.118,
668.511
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "600"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "hms",
"value": 508.777,
"ci": [
460.002,
565.727
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "509"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 88406.225,
"ci": [
87276.419,
94275.323
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "88,406"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "lucene",
"value": 1754.898,
"ci": [
1671.394,
2145.252
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,755"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "tantivy",
"value": 1269.343,
"ci": [
1093.41,
1423.457
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,269"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hms",
"value": 2.156009168,
"ci": [
2.131987949,
2.188588237
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2.16"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 1.1806630159990164,
"ci": [
1.1729135619989393,
1.1811330580003414
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1.18"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "lucene",
"value": 2.630518981,
"ci": [
2.473350822,
2.778356151
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2.63"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "tantivy",
"value": 2.6500262409990682,
"ci": [
2.6409532399993623,
2.6505701259993657
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2.65"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "hms",
"value": 20.562817573547363,
"ci": [
20.562817573547363,
20.562817573547363
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "20.6"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 27.0859375,
"ci": [
27.0859375,
27.0859375
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "27.1"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "lucene",
"value": 12.16652774810791,
"ci": [
12.16652774810791,
12.16652774810791
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "12.2"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "tantivy",
"value": 17.35812759399414,
"ci": [
17.35812759399414,
17.35812759399414
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "17.4"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.840599199,
"ci": [
0.836939676,
0.942827962
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.84"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.081541718,
"ci": [
0.077916238,
0.088280834
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.08"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.291951304,
"ci": [
0.275507479,
0.31941395
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.29"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.024903889,
"ci": [
0.023638664,
0.02968958
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.02"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 1.71895986,
"ci": [
1.71895986,
1.71895986
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1.719"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 9.141574070017668,
"ci": [
9.141574070017668,
9.141574070017668
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "9.142"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 31.037972089999997,
"ci": [
31.037972089999997,
31.037972089999997
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "31.038"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 82.71144980999452,
"ci": [
82.71144980999452,
82.71144980999452
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "82.711"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 3.75641631,
"ci": [
3.75641631,
3.75641631
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "3.756"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 8.763046479998593,
"ci": [
8.763046479998593,
8.763046479998593
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "8.763"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 14.948087370000001,
"ci": [
14.948087370000001,
14.948087370000001
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "14.948"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 43.996110599982785,
"ci": [
43.996110599982785,
43.996110599982785
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "43.996"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hms",
"value": 2.4908218383789062,
"ci": [
2.4908218383789062,
2.4908218383789062
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.5"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "beir:fiqa (57,600 documents, 500 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.4561419499447157,
"ci": [
0.34378214839170335,
0.5689042043949878
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4561"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms-eager",
"value": 0.4561419499447157,
"ci": [
0.34378214839170335,
0.5689042043949878
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4561"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.45009623895127726,
"ci": [
0.3322770359003301,
0.5667724678193115
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4501"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.46265299364569407,
"ci": [
0.3452561205276061,
0.5757421844639584
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4627"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.4491250108653578,
"ci": [
0.3343985674609109,
0.5627609557586442
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4491"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "whoosh",
"value": 0.4401216437164885,
"ci": [
0.3277974760341089,
0.5514643172818812
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4401"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "bm25s",
"value": 0.44175557833638,
"ci": [
0.3275175531106332,
0.5507361325996123
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.4418"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 0.9859267230723582,
"factor_infinite": false,
"ci": [
0.597111272490446,
1.6477744218570605
],
"verdict": "TIE",
"text": "0.99x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms",
"value": 0.8203333333333334,
"ci": [
0.7254999999999999,
0.905
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8203"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms-eager",
"value": 0.8203333333333334,
"ci": [
0.7254999999999999,
0.905
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8203"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.8391666666666667,
"ci": [
0.7396666666666667,
0.919
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8392"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "lucene",
"value": 0.8236666666666665,
"ci": [
0.7254999999999999,
0.905
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8237"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.8203333333333334,
"ci": [
0.7254999999999999,
0.905
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8203"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "whoosh",
"value": 0.8206666666666667,
"ci": [
0.7238333333333334,
0.909
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8207"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "bm25s",
"value": 0.8311666666666666,
"ci": [
0.733,
0.915
],
"n": 100,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.8312"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.9775571002979145,
"factor_infinite": false,
"ci": [
0.7894450489662675,
1.2235241099594412
],
"verdict": "TIE",
"text": "0.98x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms",
"value": 2288.679292332942,
"ci": [
2118.0838077046783,
2365.9257667604606
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,289"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms-eager",
"value": 2232.159244574398,
"ci": [
2072.383935982237,
2305.5221197210144
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,232"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 32.425558442954014,
"ci": [
31.981502390253354,
32.80521041929057
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "32"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "lucene",
"value": 1002.2853725087555,
"ci": [
958.5533420939997,
1019.166381826649
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,002"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "tantivy",
"value": 2503.8034244261294,
"ci": [
2269.622326220352,
2556.782210084709
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,504"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "whoosh",
"value": 13.599709372345025,
"ci": [
13.408249665024112,
13.69029410190459
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "14"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "bm25s",
"value": 1102.183527988817,
"ci": [
1029.9761689263914,
1149.0237365783112
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,102"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.914081061638258,
"factor_infinite": false,
"ci": [
0.8284177664215303,
1.0424314827306465
],
"verdict": "TIE",
"text": "0.91x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms",
"value": 420.2165,
"ci": [
401.735,
446.873
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "420"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms-eager",
"value": 431.558,
"ci": [
411.905,
461.791
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "432"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 24740.3205,
"ci": [
24537.211,
25147.863
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "24,740"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "lucene",
"value": 930.0550000000001,
"ci": [
916.458,
947.398
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "930"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "tantivy",
"value": 379.59749999999997,
"ci": [
372.469,
416.904
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "380"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "whoosh",
"value": 68913.053,
"ci": [
67785.599,
69688.196
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "68,913"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "bm25s",
"value": 899.6585,
"ci": [
848.974,
947.889
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "900"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.9033379222377036,
"factor_infinite": false,
"ci": [
0.8335007932902637,
1.0377587215452972
],
"verdict": "TIE",
"text": "0.90x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms",
"value": 774.961,
"ci": [
725.49,
802.218
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "775"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms-eager",
"value": 780.2764999999999,
"ci": [
731.141,
815.193
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "780"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 118386.10500000001,
"ci": [
117000.749,
119354.506
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "118,386"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "lucene",
"value": 2212.803,
"ci": [
1854.138,
2609.766
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,213"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "tantivy",
"value": 720.3705,
"ci": [
692.567,
741.16
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "720"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "whoosh",
"value": 131749.15350000001,
"ci": [
129717.812,
132579.185
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "131,749"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "bm25s",
"value": 1249.0225,
"ci": [
1130.652,
1301.768
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,249"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.9295571002927889,
"factor_infinite": false,
"ci": [
0.8633152085842004,
1.0215991950268095
],
"verdict": "TIE",
"text": "0.93x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms",
"value": 2.515448795,
"ci": [
2.452041575,
2.578856015
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2.52"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms-eager",
"value": 16.573853896499998,
"ci": [
16.402460064,
16.745247729
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "16.57"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 2.0610938575000546,
"ci": [
2.032452352000064,
2.0897353630000453
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2.06"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "lucene",
"value": 4.0917769785,
"ci": [
4.042061258,
4.141492699
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "4.09"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "tantivy",
"value": 3.476251229000013,
"ci": [
3.4579393139999866,
3.4945631440000398
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "3.48"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "whoosh",
"value": 41.11348974550003,
"ci": [
40.822708994000095,
41.40427049699997
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "41.11"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "bm25s",
"value": 5.152338578999888,
"ci": [
5.13049070399984,
5.174186453999937
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "5.15"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.8193742053493299,
"factor_infinite": false,
"ci": [
0.7881216865843764,
0.8522430387421328
],
"verdict": "LOSS",
"text": "0.82x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": 132.7890625,
"ci": [
132.38671875,
133.19140625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "133"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms-eager",
"value": 166.75390625,
"ci": [
165.9453125,
167.5625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "167"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": 120.546875,
"ci": [
120.5390625,
120.5546875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "121"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": 768.109375,
"ci": [
764.265625,
771.953125
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "768"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": 244.4921875,
"ci": [
244.26171875,
244.72265625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "244"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "whoosh",
"value": 536.255859375,
"ci": [
536.08984375,
536.421875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "536"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "bm25s",
"value": 236.708984375,
"ci": [
236.12890625,
237.2890625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "237"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.9078072601047243,
"factor_infinite": false,
"ci": [
0.9050063055400769,
0.9106252397391638
],
"verdict": "LOSS",
"text": "0.91x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms",
"value": 65.912109375,
"ci": [
64.90234375,
66.77734375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "66"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms-eager",
"value": 74.197265625,
"ci": [
73.5625,
75.1015625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "74"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": 25.828125,
"ci": [
25.05078125,
26.326171875
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "26"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": 438.154296875,
"ci": [
434.078125,
439.94140625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "438"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": 37.724609375,
"ci": [
37.06640625,
38.5703125
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "38"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "whoosh",
"value": 141.31640625,
"ci": [
140.01953125,
143.0390625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "141"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "bm25s",
"value": 83.640625,
"ci": [
83.04296875,
84.0703125
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "84"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.39185705396035203,
"factor_infinite": false,
"ci": [
0.3751389295115531,
0.40562744507974724
],
"verdict": "LOSS",
"text": "0.39x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms",
"value": 9.793286323547363,
"ci": [
9.793286323547363,
9.793286323547363
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "9.8"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms-eager",
"value": 9.785473823547363,
"ci": [
9.785473823547363,
9.785473823547363
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "9.8"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 63.0390625,
"ci": [
63.0390625,
63.0390625
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "63.0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "lucene",
"value": 10.313970565795898,
"ci": [
10.313970565795898,
10.313970565795898
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "10.3"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "tantivy",
"value": 13.706984519958496,
"ci": [
13.706984519958496,
13.706984519958496
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "13.7"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "whoosh",
"value": 48.566134452819824,
"ci": [
48.566134452819824,
48.566134452819824
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "48.6"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "bm25s",
"value": 14.216367721557617,
"ci": [
14.216367721557617,
14.216367721557617
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "14.2"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 1.053167468513259,
"factor_infinite": false,
"ci": null,
"verdict": "WIN",
"text": "1.05x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.619563115,
"ci": [
0.594086714,
0.641814565
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.62"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms-eager",
"value": 0.6272474809999999,
"ci": [
0.591178799,
0.649707937
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.63"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.08885403750000001,
"ci": [
0.086029671,
0.092307714
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.09"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.4474606435,
"ci": [
0.433207579,
0.479147153
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.45"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.0718216415,
"ci": [
0.065463099,
0.077803712
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.07"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "whoosh",
"value": 0.289982277,
"ci": [
0.279255388,
0.305334852
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.29"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "bm25s",
"value": 0.24022912549999997,
"ci": [
0.224627369,
0.248742769
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.24"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.11592304280412175,
"factor_infinite": false,
"ci": [
0.10199690466669294,
0.1309635616594516
],
"verdict": "LOSS",
"text": "0.12x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 3.9187444300000003,
"ci": [
3.9187444300000003,
3.9187444300000003
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.919"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms-eager",
"value": 3.8050166800000005,
"ci": [
3.8050166800000005,
3.8050166800000005
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.805"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 4.625860979999743,
"ci": [
4.625860979999743,
4.625860979999743
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "4.626"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 18.03878455,
"ci": [
18.03878455,
18.03878455
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "18.039"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 163.8972191999983,
"ci": [
163.8972191999983,
163.8972191999983
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "163.897"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "whoosh",
"value": 146.63095731999874,
"ci": [
146.63095731999874,
146.63095731999874
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "146.631"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 1.1804446711519134,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "1.18x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 3.1620943699999997,
"ci": [
3.1620943699999997,
3.1620943699999997
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.162"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms-eager",
"value": 3.2690420799999997,
"ci": [
3.2690420799999997,
3.2690420799999997
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.269"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 6.417760269996506,
"ci": [
6.417760269996506,
6.417760269996506
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "6.418"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 1.48195727,
"ci": [
1.48195727,
1.48195727
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.482"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 176.53093911999804,
"ci": [
176.53093911999804,
176.53093911999804
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "176.531"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "whoosh",
"value": 16.973000140001204,
"ci": [
16.973000140001204,
16.973000140001204
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "16.973"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 0.4686632012187543,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.47x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms-eager",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "whoosh",
"value": 1.9728384017944336,
"ci": [
1.9728384017944336,
1.9728384017944336
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "bm25s",
"value": 56.54034996032715,
"ci": [
56.54034996032715,
56.54034996032715
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "56.5"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "whoosh",
"factor": 0.8301754360619107,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.83x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms-eager",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "whoosh",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "bm25s",
"value": 3,
"ci": [
3,
3
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.010869565217391304,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.01x"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.4561419499447157,
"ci": [
0.37855253576415726,
0.5350347701184234
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.4561"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.45009623895127726,
"ci": [
0.3664069435962895,
0.5334244252772741
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.4501"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.46265299364569407,
"ci": [
0.38441067792846956,
0.5426832663257491
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.4627"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.4491250108653578,
"ci": [
0.372011717364413,
0.5296541538656578
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.4491"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "hms",
"value": 0.8203333333333334,
"ci": [
0.7538333333333334,
0.8826666666666667
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.8203"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.8391666666666667,
"ci": [
0.7718333333333334,
0.8998333333333334
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.8392"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "lucene",
"value": 0.8236666666666665,
"ci": [
0.7573333333333333,
0.8858333333333333
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.8237"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.8203333333333334,
"ci": [
0.7538333333333334,
0.8826666666666667
],
"n": 100,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.8203"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "hms",
"value": 12628.515668262533,
"ci": [
11050.469816249473,
12771.509520904918
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "12,629"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 62.59960277572298,
"ci": [
53.23821643099081,
63.0343365952096
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "63"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "lucene",
"value": 1338.3452757897326,
"ci": [
1154.2618967840313,
1367.5202080512422
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,338"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "tantivy",
"value": 4849.943241114249,
"ci": [
4503.816556669204,
5298.197945125717
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "4,850"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "hms",
"value": 75.674,
"ci": [
74.161,
82.426
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "76"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 12431.824,
"ci": [
12368.788,
14777.976
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "12,432"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "lucene",
"value": 688.722,
"ci": [
663.013,
819.938
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "689"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "tantivy",
"value": 190.226,
"ci": [
179.244,
211.096
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "190"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "hms",
"value": 144.633,
"ci": [
142.803,
174.308
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "145"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 61156.511,
"ci": [
59640.913,
66734.402
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "61,157"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "lucene",
"value": 1365.833,
"ci": [
1346.877,
1477.334
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,366"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "tantivy",
"value": 396.206,
"ci": [
336.25,
449.623
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "396"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hms",
"value": 1.213714217,
"ci": [
1.196267904,
1.2146892440000001
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1.21"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 0.8815983789991151,
"ci": [
0.8279929869986518,
0.9001539719993161
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.88"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "lucene",
"value": 2.25939896,
"ci": [
2.248518259,
2.347525725
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2.26"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "tantivy",
"value": 1.7403191430003062,
"ci": [
1.734949711999434,
1.7461702269993111
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1.74"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "hms",
"value": 9.797192573547363,
"ci": [
9.797192573547363,
9.797192573547363
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "9.8"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 19.25390625,
"ci": [
19.25390625,
19.25390625
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "19.3"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "lucene",
"value": 10.313986778259277,
"ci": [
10.313986778259277,
10.313986778259277
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "10.3"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "tantivy",
"value": 13.706984519958496,
"ci": [
13.706984519958496,
13.706984519958496
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "13.7"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.301695743,
"ci": [
0.287584375,
0.317913052
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.30"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.035247877,
"ci": [
0.031886552,
0.038362907
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.04"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.275996363,
"ci": [
0.271379881,
0.316352287
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.28"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.022639198,
"ci": [
0.020162576,
0.022888295
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.02"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 4.456856650000001,
"ci": [
4.456856650000001,
4.456856650000001
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "4.457"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 11.852811349999683,
"ci": [
11.852811349999683,
11.852811349999683
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "11.853"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 38.05683817,
"ci": [
38.05683817,
38.05683817
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "38.057"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 93.66708319999816,
"ci": [
93.66708319999816,
93.66708319999816
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "93.667"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 3.3314022100000003,
"ci": [
3.3314022100000003,
3.3314022100000003
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "3.331"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 10.455956980003975,
"ci": [
10.455956980003975,
10.455956980003975
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "10.456"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 17.638506319999998,
"ci": [
17.638506319999998,
17.638506319999998
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "17.639"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 52.88391140000385,
"ci": [
52.88391140000385,
52.88391140000385
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "52.884"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hms",
"value": 2.4908218383789062,
"ci": [
2.4908218383789062,
2.4908218383789062
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.5"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "memory (4,559 documents, 100 queries)",
"file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.29158226275481797,
"ci": [
0.25912963977629083,
0.32356876108307125
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2916"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.29140981016338025,
"ci": [
0.2580388719165204,
0.32356280551113314
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2914"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.28972271928920523,
"ci": [
0.2558415031346977,
0.321223315571466
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2897"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.29006819680247453,
"ci": [
0.2567207176285045,
0.3220969741370192
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2901"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "hms",
"value": 0.22742500506264265,
"ci": [
0.19875352037951716,
0.25456973325714316
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2274"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.22713744545849535,
"ci": [
0.19870144179042648,
0.2541558791606999
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2271"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "lucene",
"value": 0.22757113786424402,
"ci": [
0.19879553549983092,
0.25449097683254396
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2276"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.22689859592232703,
"ci": [
0.1983430475484094,
0.25373280688143784
],
"n": 324,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.2269"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "hms",
"value": 40784.21499826919,
"ci": [
37478.85741307508,
41541.912712748446
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "40,784"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 1415.6626698169298,
"ci": [
1359.7652789962624,
1453.5804663585916
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "1,416"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "lucene",
"value": 4114.5853605973725,
"ci": [
3918.9511551362534,
4517.162897691404
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "4,115"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "tantivy",
"value": 15413.471364458048,
"ci": [
14543.941714345583,
15804.180352038118
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "15,413"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "hms",
"value": 25.791,
"ci": [
25.225,
27.906
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "26"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 254.805,
"ci": [
247.493,
256.333
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "255"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "lucene",
"value": 283.067,
"ci": [
263.002,
297.919
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "283"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "tantivy",
"value": 70.693,
"ci": [
68.494,
75.971
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "71"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "hms",
"value": 57.175,
"ci": [
53.701,
64.058
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "57"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 2701.062,
"ci": [
2635.155,
2875.719
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2,701"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "lucene",
"value": 539.166,
"ci": [
471.91,
567.596
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "539"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "tantivy",
"value": 149.59,
"ci": [
140.325,
156.595
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "150"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hms",
"value": 0.230839372,
"ci": [
0.23019802,
0.232152894
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.23"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 0.13541231199997128,
"ci": [
0.1334877620010957,
0.14226272999985667
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.14"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "lucene",
"value": 0.619037672,
"ci": [
0.614903845,
0.621166554
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.62"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "tantivy",
"value": 0.2790714090006077,
"ci": [
0.27693536700098775,
0.2895067500012374
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.28"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hms",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "invalid",
"validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix",
"uncertainty": "sampled",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "hms",
"value": 2.0628175735473633,
"ci": [
2.0628175735473633,
2.0628175735473633
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "2.1"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 4.0234375,
"ci": [
4.0234375,
4.0234375
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "4.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "lucene",
"value": 1.4865550994873047,
"ci": [
1.4865550994873047,
1.4865550994873047
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "1.5"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "tantivy",
"value": 1.9688215255737305,
"ci": [
1.9688215255737305,
1.9688215255737305
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.08889909,
"ci": [
0.083722021,
0.097128282
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.09"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.021537669,
"ci": [
0.020218705,
0.02514792
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.02"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.283127535,
"ci": [
0.272270907,
0.30104404
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.28"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.020183307,
"ci": [
0.019923964,
0.025963645
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.02"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 2.03304533,
"ci": [
2.03304533,
2.03304533
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.033"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 9.737997880001785,
"ci": [
9.737997880001785,
9.737997880001785
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "9.738"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 30.87565001,
"ci": [
30.87565001,
30.87565001
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "30.876"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 84.13805727999716,
"ci": [
84.13805727999716,
84.13805727999716
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "84.138"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 1.7135118999999999,
"ci": [
1.7135118999999999,
1.7135118999999999
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1.714"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 9.43153046999214,
"ci": [
9.43153046999214,
9.43153046999214
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "9.432"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 14.462594180000002,
"ci": [
14.462594180000002,
14.462594180000002
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "14.463"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 42.87158723000175,
"ci": [
42.87158723000175,
42.87158723000175
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "42.872"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hms",
"value": 2.4908218383789062,
"ci": [
2.4908218383789062,
2.4908218383789062
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.5"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.29158226275481797,
"ci": [
0.2389921791965676,
0.33934188094051343
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2916"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms-eager",
"value": 0.29158226275481797,
"ci": [
0.2389921791965676,
0.33934188094051343
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2916"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.29140981016338025,
"ci": [
0.23913017727550598,
0.3404583607962663
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2914"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.28972271928920523,
"ci": [
0.23724687603899755,
0.3387393508400861
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2897"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.29006819680247453,
"ci": [
0.23766263377150765,
0.3373278424145376
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2901"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "whoosh",
"value": 0.2873130912847506,
"ci": [
0.23469301865019682,
0.33341568767350244
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2873"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "bm25s",
"value": 0.287785039883203,
"ci": [
0.2346829683464342,
0.334993878551926
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2878"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 1.0005917871856855,
"factor_infinite": false,
"ci": [
0.7019718318493078,
1.419067575689336
],
"verdict": "TIE",
"text": "1.00x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms",
"value": 0.22742500506264265,
"ci": [
0.18530820446651203,
0.2699444227525327
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2274"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms-eager",
"value": 0.22742500506264265,
"ci": [
0.18530820446651203,
0.2699444227525327
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2274"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.22713744545849535,
"ci": [
0.18639467777258142,
0.2676392828738551
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2271"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "lucene",
"value": 0.22757113786424402,
"ci": [
0.18623200296625053,
0.2686162528083917
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2276"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.22689859592232703,
"ci": [
0.18437031563892306,
0.26936672640729253
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2269"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "whoosh",
"value": 0.2251819508302511,
"ci": [
0.18545926318943576,
0.26578182472231626
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2252"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "bm25s",
"value": 0.22715065794318523,
"ci": [
0.18544371473099353,
0.2681726673825259
],
"n": 324,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.2272"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 0.9993578588085782,
"factor_infinite": false,
"ci": [
0.6898622199107787,
1.4495060916111866
],
"verdict": "TIE",
"text": "1.00x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms",
"value": 10571.528830237587,
"ci": [
8628.496028907804,
11081.766471882958
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "10,572"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms-eager",
"value": 10809.300798015202,
"ci": [
10166.927455677765,
11110.307556974018
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "10,809"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 486.387220762379,
"ci": [
477.8881513725941,
493.73192764948027
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "486"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "lucene",
"value": 3022.570796310708,
"ci": [
2952.7388448123756,
3069.127289581983
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "3,023"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "tantivy",
"value": 7151.092941690469,
"ci": [
6813.435354293591,
7379.63650041182
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "7,151"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "whoosh",
"value": 131.37953514213092,
"ci": [
130.39154395383645,
132.51555600623206
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "131"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "bm25s",
"value": 1232.7320529292738,
"ci": [
1177.8034777243208,
1288.3462870911721
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,233"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 1.478309527849396,
"factor_infinite": false,
"ci": [
1.1692304937277456,
1.6264580047566772
],
"verdict": "WIN",
"text": "1.48x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms",
"value": 59.5155,
"ci": [
56.209,
78.001
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "60"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms-eager",
"value": 57.0405,
"ci": [
55.146,
62.61
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "57"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 774.52,
"ci": [
758.193,
784.624
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "775"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "lucene",
"value": 405.8085,
"ci": [
395.864,
410.793
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "406"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "tantivy",
"value": 156.923,
"ci": [
155.62,
164.648
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "157"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "whoosh",
"value": 2968.8845,
"ci": [
2931.569,
3050.117
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,969"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "bm25s",
"value": 798.5985000000001,
"ci": [
775.367,
843.713
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "799"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 2.636674479757374,
"factor_infinite": false,
"ci": [
1.9951026268893988,
2.9292106246330656
],
"verdict": "WIN",
"text": "2.64x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms",
"value": 279.0985,
"ci": [
262.466,
334.495
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "279"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms-eager",
"value": 274.59000000000003,
"ci": [
261.846,
303.196
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "275"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 6962.541499999999,
"ci": [
6752.032,
7127.956
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "6,963"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "lucene",
"value": 703.3555,
"ci": [
679.512,
740.936
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "703"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "tantivy",
"value": 293.677,
"ci": [
280.491,
317.212
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "294"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "whoosh",
"value": 33136.378,
"ci": [
32623.342,
33851.841
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "33,136"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "bm25s",
"value": 1082.0439999999999,
"ci": [
1032.613,
1189.952
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,082"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 1.052234247048981,
"factor_infinite": false,
"ci": [
0.83855065098133,
1.2085832069677596
],
"verdict": "TIE",
"text": "1.05x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms",
"value": 0.5921563255,
"ci": [
0.58976631,
0.594546341
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.59"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms-eager",
"value": 2.179494596,
"ci": [
2.170529804,
2.188459388
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2.18"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 0.285780587999966,
"ci": [
0.2805567420000443,
0.2910044339998876
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.29"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "lucene",
"value": 0.966967628,
"ci": [
0.965895586,
0.96803967
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.97"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "tantivy",
"value": 0.485183013999972,
"ci": [
0.4764130700000351,
0.4939529579999089
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.49"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "whoosh",
"value": 7.334148889499943,
"ci": [
7.328755410999975,
7.339542367999911
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "7.33"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "bm25s",
"value": 0.8843612315000655,
"ci": [
0.8741987220000738,
0.8945237410000573
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.88"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.48261004010834635,
"factor_infinite": false,
"ci": [
0.4718837248719092,
0.4934232916761346
],
"verdict": "LOSS",
"text": "0.48x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": 57.302734375,
"ci": [
56.78125,
57.82421875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "57"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms-eager",
"value": 86.779296875,
"ci": [
86.75390625,
86.8046875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "87"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": 33.82421875,
"ci": [
33.7890625,
33.859375
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "34"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": 454.845703125,
"ci": [
448.98828125,
460.703125
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "455"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": 77.69140625,
"ci": [
75.26171875,
80.12109375
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "78"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "whoosh",
"value": 144.1953125,
"ci": [
143.53125,
144.859375
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "144"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "bm25s",
"value": 91.8046875,
"ci": [
91.703125,
91.90625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "92"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.5902723337537067,
"factor_infinite": false,
"ci": [
0.5843410119570357,
0.5963126031920748
],
"verdict": "LOSS",
"text": "0.59x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms",
"value": 56.791015625,
"ci": [
56.17578125,
57.99609375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "57"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms-eager",
"value": 58.37109375,
"ci": [
57.640625,
58.87890625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "58"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": 29.9296875,
"ci": [
28.984375,
30.24609375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "30"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": 441.1484375,
"ci": [
435.97265625,
444.203125
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "441"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": 41.001953125,
"ci": [
39.96484375,
41.5078125
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "41"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "whoosh",
"value": 69.625,
"ci": [
68.90625,
70.130859375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "70"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "bm25s",
"value": 70.138671875,
"ci": [
69.5078125,
70.66796875
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "70"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.5270144787976752,
"factor_infinite": false,
"ci": [
0.49976426214049974,
0.5384187469577916
],
"verdict": "LOSS",
"text": "0.53x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms",
"value": 2.0628175735473633,
"ci": [
2.0628175735473633,
2.0628175735473633
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.1"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms-eager",
"value": 2.0628175735473633,
"ci": [
2.0628175735473633,
2.0628175735473633
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.1"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 9.3046875,
"ci": [
9.3046875,
9.3046875
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "9.3"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "lucene",
"value": 1.4865388870239258,
"ci": [
1.4865388870239258,
1.4865388870239258
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "1.5"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "tantivy",
"value": 1.9688215255737305,
"ci": [
1.9688215255737305,
1.9688215255737305
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "whoosh",
"value": 9.888919830322266,
"ci": [
9.888919830322266,
9.888919830322266
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "9.9"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "bm25s",
"value": 3.3759756088256836,
"ci": [
3.3759756088256836,
3.3759756088256836
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "3.4"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 0.7206351672036472,
"factor_infinite": false,
"ci": null,
"verdict": "LOSS",
"text": "0.72x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.175434672,
"ci": [
0.170015857,
0.187209304
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.18"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms-eager",
"value": 0.181669582,
"ci": [
0.176235174,
0.189533834
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.18"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.068488589,
"ci": [
0.065624187,
0.06911327
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.07"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.4379882085,
"ci": [
0.418019085,
0.447336709
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.44"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.068223135,
"ci": [
0.065778959,
0.072269295
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.07"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "whoosh",
"value": 0.241962255,
"ci": [
0.231076263,
0.258533869
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.24"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "bm25s",
"value": 0.20877574,
"ci": [
0.199165031,
0.219764162
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.21"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.38888056860276743,
"factor_infinite": false,
"ci": [
0.35136586480765936,
0.4250738506114756
],
"verdict": "LOSS",
"text": "0.39x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 1.07312056,
"ci": [
1.07312056,
1.07312056
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.073"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms-eager",
"value": 1.06455452,
"ci": [
1.06455452,
1.06455452
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.065"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 3.140645449998374,
"ci": [
3.140645449998374,
3.140645449998374
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.141"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 18.21623659,
"ci": [
18.21623659,
18.21623659
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "18.216"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 55.32423113999812,
"ci": [
55.32423113999812,
55.32423113999812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "55.324"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "whoosh",
"value": 53.212382009999146,
"ci": [
53.212382009999146,
53.212382009999146
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "53.212"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 2.9266473563775293,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "2.93x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 0.72923147,
"ci": [
0.72923147,
0.72923147
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0.729"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms-eager",
"value": 0.7055342400000001,
"ci": [
0.7055342400000001,
0.7055342400000001
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0.706"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 8.833210899999813,
"ci": [
8.833210899999813,
8.833210899999813
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "8.833"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 1.3494846699999998,
"ci": [
1.3494846699999998,
1.3494846699999998
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.349"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 50.4753322800002,
"ci": [
50.4753322800002,
50.4753322800002
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "50.475"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "whoosh",
"value": 6.140949739999542,
"ci": [
6.140949739999542,
6.140949739999542
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "6.141"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 1.8505573682934993,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "1.85x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms-eager",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "whoosh",
"value": 1.9728384017944336,
"ci": [
1.9728384017944336,
1.9728384017944336
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "bm25s",
"value": 56.54034996032715,
"ci": [
56.54034996032715,
56.54034996032715
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "56.5"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "whoosh",
"factor": 0.8301754360619107,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.83x"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms-eager",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "whoosh",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "bm25s",
"value": 3,
"ci": [
3,
3
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3"
},
{
"section": "beir:nfcorpus (3,633 documents, 324 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.010869565217391304,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.01x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.7026997341473034,
"ci": [
0.6767786729633418,
0.7322695862845088
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.7027"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.7009278481912261,
"ci": [
0.6753581453218557,
0.7306336500865396
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.7009"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.7021782838562147,
"ci": [
0.6765209168126257,
0.7322968706728786
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.7022"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.701325671999324,
"ci": [
0.6755941449618927,
0.7308008746966628
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.7013"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "hms",
"value": 0.9264112072517512,
"ci": [
0.9080758137618459,
0.9446847960444994
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.9264"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.9321796456530697,
"ci": [
0.9157601977750309,
0.9490111248454882
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.9322"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "lucene",
"value": 0.9295014421096004,
"ci": [
0.9130613926658425,
0.9474660074165636
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.9295"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.9282653481664607,
"ci": [
0.9110630407911001,
0.9466007416563659
],
"n": 809,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.9283"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "hms",
"value": 17781.668497694965,
"ci": [
16670.54615758954,
17880.6832834575
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "17,782"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 242.16717008571558,
"ci": [
239.0382548446373,
243.66628093250253
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "242"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "lucene",
"value": 2862.2097821881707,
"ci": [
2518.397806522205,
2929.936108270389
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "2,862"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"system": "tantivy",
"value": 6450.044975709162,
"ci": [
6418.39423883029,
6507.677325740711
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "6,450"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "hms",
"value": 55.432,
"ci": [
54.937,
56.628
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "55"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 3852.758,
"ci": [
3840.981,
3879.967
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "3,853"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "lucene",
"value": 329.247,
"ci": [
319.601,
368.767
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "329"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"system": "tantivy",
"value": 150.532,
"ci": [
150.181,
150.772
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "151"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "hms",
"value": 94.551,
"ci": [
81.733,
103.839
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "95"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 9342.801,
"ci": [
9318.986,
9672.006
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "9,343"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "lucene",
"value": 789.441,
"ci": [
717.179,
865.042
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "789"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"system": "tantivy",
"value": 287.282,
"ci": [
242.068,
319.241
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "287"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "hms",
"value": 0.312045625,
"ci": [
0.310915802,
0.317366839
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.31"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 0.19639155000004394,
"ci": [
0.1934057339985884,
0.21986437700070383
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.20"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "lucene",
"value": 0.712891089,
"ci": [
0.682538507,
0.789371421
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.71"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"system": "tantivy",
"value": 0.36752783200063277,
"ci": [
0.3633899329997803,
0.3676003689997742
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.37"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": 73.18359375,
"ci": [
73.18359375,
73.18359375
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "73"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": 73.18359375,
"ci": [
73.18359375,
73.18359375
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "73"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": 162.078125,
"ci": [
158.83984375,
167.75
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "162"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": 73.18359375,
"ci": [
73.18359375,
73.18359375
],
"n": 3,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "73"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "build_peak_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "hms",
"value": 195.8359375,
"ci": [
143.99609375,
329.3828125
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "196"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": 218.0078125,
"ci": [
84.171875,
351.5546875
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "218"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": 410.25390625,
"ci": [
403.73046875,
426.3046875
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "410"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": 173.6640625,
"ci": [
84.859375,
396.0703125
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "174"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "serving_rss_mib",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "hms",
"value": 2.9104738235473633,
"ci": [
2.9104738235473633,
2.9104738235473633
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "2.9"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 5.87109375,
"ci": [
5.87109375,
5.87109375
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "5.9"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "lucene",
"value": 1.9866275787353516,
"ci": [
1.9866275787353516,
1.9866275787353516
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"system": "tantivy",
"value": 2.6494836807250977,
"ci": [
2.6494836807250977,
2.6494836807250977
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "2.6"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.117377458,
"ci": [
0.11571311,
0.119997146
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.12"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.026888259,
"ci": [
0.023155602,
0.029468671
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.03"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.291827871,
"ci": [
0.28430985,
2.911798323
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.29"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.024623582,
"ci": [
0.023976055,
0.025707532
],
"n": 5,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "sampled",
"text": "0.02"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 2.00371566,
"ci": [
2.00371566,
2.00371566
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.004"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 9.131574970015208,
"ci": [
9.131574970015208,
9.131574970015208
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "9.132"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 30.77421369,
"ci": [
30.77421369,
30.77421369
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "30.774"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 86.06415542000832,
"ci": [
86.06415542000832,
86.06415542000832
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "86.064"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 1.73026945,
"ci": [
1.73026945,
1.73026945
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1.730"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 8.99912264001614,
"ci": [
8.99912264001614,
8.99912264001614
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "8.999"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 14.38683205,
"ci": [
14.38683205,
14.38683205
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "14.387"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 44.8263032900104,
"ci": [
44.8263032900104,
44.8263032900104
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "44.826"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "hms",
"value": 2.4908218383789062,
"ci": [
2.4908218383789062,
2.4908218383789062
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2.5"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install",
"validity": "unverified",
"validity_reason": "source revision is not pinned",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json",
"hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": null,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "n/v"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms",
"value": 0.7026997341473034,
"ci": [
0.6664189500472893,
0.7419550353136701
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7027"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "hms-eager",
"value": 0.7026997341473034,
"ci": [
0.6664189500472893,
0.7419550353136701
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7027"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "sqlite-fts5",
"value": 0.7009278481912261,
"ci": [
0.6643714062414602,
0.7404343587518659
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7009"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "lucene",
"value": 0.7021782838562147,
"ci": [
0.6659037403180936,
0.741114931390745
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7022"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "tantivy",
"value": 0.701325671999324,
"ci": [
0.6640624257389993,
0.7411381074045522
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7013"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "whoosh",
"value": 0.6907626470247572,
"ci": [
0.6518433173669151,
0.7289895253481598
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.6908"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"system": "bm25s",
"value": 0.700081542866468,
"ci": [
0.6619669702121025,
0.737860727292892
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.7001"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "ndcg@10",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 1.0007426180830101,
"factor_infinite": false,
"ci": [
0.8992113393218454,
1.114207640520607
],
"verdict": "TIE",
"text": "1.00x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms",
"value": 0.9264112072517512,
"ci": [
0.8996909765142151,
0.9501030078285949
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9264"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "hms-eager",
"value": 0.9264112072517512,
"ci": [
0.8996909765142151,
0.9501030078285949
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9264"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "sqlite-fts5",
"value": 0.9321796456530697,
"ci": [
0.9083848372476309,
0.9558096415327565
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9322"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "lucene",
"value": 0.9295014421096004,
"ci": [
0.9023691800576844,
0.9522661722290894
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9295"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "tantivy",
"value": 0.9282653481664607,
"ci": [
0.9024103831891224,
0.9524103831891224
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9283"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "whoosh",
"value": 0.9185826122785331,
"ci": [
0.888710341985991,
0.9443757725587144
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9186"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"system": "bm25s",
"value": 0.9282653481664607,
"ci": [
0.9009064688916358,
0.9501030078285949
],
"n": 809,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.9283"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "recall@100",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.9938118811881188,
"factor_infinite": false,
"ci": [
0.9412867765923053,
1.0459256571337854
],
"verdict": "TIE",
"text": "0.99x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms",
"value": 2732.645695786485,
"ci": [
2632.5697276850024,
2808.922891816741
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,733"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "hms-eager",
"value": 2701.8609694681927,
"ci": [
2611.901411279101,
2782.887516243859
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,702"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "sqlite-fts5",
"value": 99.47137943955144,
"ci": [
97.34158790255437,
102.312572751274
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "99"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "lucene",
"value": 1457.3055473838826,
"ci": [
1421.054644223512,
1540.472346295411
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,457"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "tantivy",
"value": 2964.8401183361775,
"ci": [
2853.0639279227885,
3029.1496653076892
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "2,965"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "whoosh",
"value": 23.438295567347772,
"ci": [
22.35560006516289,
24.001002269987797
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "23"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"system": "bm25s",
"value": 1126.786683480005,
"ci": [
1064.4720426825868,
1193.315123610056
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,127"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "qps",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.9216839987041201,
"factor_infinite": false,
"ci": [
0.8690787906042919,
0.9845285499304655
],
"verdict": "LOSS",
"text": "0.92x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms",
"value": 372.6545,
"ci": [
361.298,
385.574
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "373"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "hms-eager",
"value": 376.171,
"ci": [
367.069,
389.382
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "376"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "sqlite-fts5",
"value": 9931.3925,
"ci": [
9670.728,
10183.449
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "9,931"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "lucene",
"value": 641.6034999999999,
"ci": [
618.825,
655.475
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "642"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "tantivy",
"value": 334.0,
"ci": [
327.7325,
346.469
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "334"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "whoosh",
"value": 36022.854999999996,
"ci": [
35006.087,
37407.892
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "36,023"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"system": "bm25s",
"value": 870.7950000000001,
"ci": [
822.502,
916.248
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "871"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p50_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.8962725527264531,
"factor_infinite": false,
"ci": [
0.8499859949062956,
0.958956318606801
],
"verdict": "LOSS",
"text": "0.90x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms",
"value": 518.2855,
"ci": [
510.285,
544.661
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "518"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "hms-eager",
"value": 530.163,
"ci": [
509.442,
547.797
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "530"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "sqlite-fts5",
"value": 17709.493499999997,
"ci": [
17362.939,
18452.92
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "17,709"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "lucene",
"value": 1127.601,
"ci": [
1061.158,
1197.361
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,128"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "tantivy",
"value": 508.30050000000006,
"ci": [
473.705,
609.637
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "508"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "whoosh",
"value": 128595.2095,
"ci": [
127226.219,
137014.661
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "128,595"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"system": "bm25s",
"value": 1258.5345000000002,
"ci": [
1168.826,
1339.244
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1,259"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "p99_latency",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.9807345565330308,
"factor_infinite": false,
"ci": [
0.8697244708176279,
1.1946990407321396
],
"verdict": "TIE",
"text": "0.98x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms",
"value": 1.4185485660000001,
"ci": [
1.327840441,
1.509256691
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1.42"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "hms-eager",
"value": 3.6037260925,
"ci": [
3.56142862,
3.646023565
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "3.60"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "sqlite-fts5",
"value": 0.42722204650000606,
"ci": [
0.4247947750000094,
0.4296493180000027
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.43"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "lucene",
"value": 1.5220980705,
"ci": [
1.384241498,
1.659954643
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1.52"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "tantivy",
"value": 0.7557944099999929,
"ci": [
0.7396925189999877,
0.7718963009999982
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.76"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "whoosh",
"value": 11.381427306000006,
"ci": [
11.27367346600002,
11.489181145999993
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "11.38"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"system": "bm25s",
"value": 1.4012112320000085,
"ci": [
1.306480692000008,
1.495941772000009
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "1.40"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_seconds",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.3011684314092113,
"factor_infinite": false,
"ci": [
0.28145959367491674,
0.3235699898373578
],
"verdict": "LOSS",
"text": "0.30x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms",
"value": 65.451171875,
"ci": [
65.1484375,
65.75390625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "65"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "hms-eager",
"value": 95.69140625,
"ci": [
95.4140625,
95.96875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "96"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "sqlite-fts5",
"value": 39.20703125,
"ci": [
38.671875,
39.7421875
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "39"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "lucene",
"value": 504.421875,
"ci": [
502.16796875,
506.67578125
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "504"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "tantivy",
"value": 90.45703125,
"ci": [
84.25,
96.6640625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "90"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "whoosh",
"value": 178.310546875,
"ci": [
178.03125,
178.58984375
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "178"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"system": "bm25s",
"value": 104.4609375,
"ci": [
104.28125,
104.640625
],
"n": 2,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "104"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "build_peak_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.5990271851033989,
"factor_infinite": false,
"ci": [
0.5881304580288719,
0.6100251828756446
],
"verdict": "LOSS",
"text": "0.60x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms",
"value": 159.7421875,
"ci": [
158.28515625,
160.828125
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "160"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "hms-eager",
"value": 160.318359375,
"ci": [
158.873046875,
161.94140625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "160"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "sqlite-fts5",
"value": 54.490234375,
"ci": [
53.89453125,
54.88671875
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "54"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "lucene",
"value": 951.931640625,
"ci": [
935.44140625,
971.515625
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "952"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "tantivy",
"value": 65.037109375,
"ci": [
64.04296875,
65.734375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "65"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "whoosh",
"value": 101.42578125,
"ci": [
100.703125,
102.3984375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "101"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"system": "bm25s",
"value": 95.208984375,
"ci": [
94.09375,
96.24609375
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "95"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "serving_rss_mib",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.34111361079865016,
"factor_infinite": false,
"ci": [
0.3351063829787234,
0.346758470916315
],
"verdict": "LOSS",
"text": "0.34x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms",
"value": 2.910472869873047,
"ci": [
2.910472869873047,
2.910472869873047
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.9"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "hms-eager",
"value": 2.906566619873047,
"ci": [
2.906566619873047,
2.906566619873047
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.9"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "sqlite-fts5",
"value": 12.4375,
"ci": [
12.4375,
12.4375
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "12.4"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "lucene",
"value": 1.9866113662719727,
"ci": [
1.9866113662719727,
1.9866113662719727
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "tantivy",
"value": 2.6494836807250977,
"ci": [
2.6494836807250977,
2.6494836807250977
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.6"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "whoosh",
"value": 13.159749031066895,
"ci": [
13.159749031066895,
13.159749031066895
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "13.2"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"system": "bm25s",
"value": 4.487649917602539,
"ci": [
4.487649917602539,
4.487649917602539
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "4.5"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "disk_mib",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 0.6825734013313883,
"factor_infinite": false,
"ci": null,
"verdict": "LOSS",
"text": "0.68x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms",
"value": 0.23930724650000001,
"ci": [
0.225467531,
0.268876353
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.24"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "hms-eager",
"value": 0.2479915995,
"ci": [
0.241085405,
0.253164824
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.25"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "sqlite-fts5",
"value": 0.0766115495,
"ci": [
0.073073611,
0.08288524
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.08"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "lucene",
"value": 0.4577830215,
"ci": [
0.446489742,
0.514178286
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.46"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "tantivy",
"value": 0.080238569,
"ci": [
0.075746264,
0.085537997
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.08"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "whoosh",
"value": 0.2392416195,
"ci": [
0.226745583,
0.252238708
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.24"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"system": "bm25s",
"value": 0.23249816099999998,
"ci": [
0.229607241,
0.2436112325
],
"n": 10,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s",
"validity": "valid",
"validity_reason": "",
"uncertainty": "sampled",
"text": "0.23"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "cold_start_s",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 0.32013886173731054,
"factor_infinite": false,
"ci": [
0.2717740336205765,
0.36761497157655043
],
"verdict": "LOSS",
"text": "0.32x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms",
"value": 1.01634077,
"ci": [
1.01634077,
1.01634077
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.016"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "hms-eager",
"value": 0.9228250499999999,
"ci": [
0.9228250499999999,
0.9228250499999999
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0.923"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "sqlite-fts5",
"value": 3.199492190001365,
"ci": [
3.199492190001365,
3.199492190001365
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3.199"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "lucene",
"value": 16.97920893,
"ci": [
16.97920893,
16.97920893
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "16.979"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "tantivy",
"value": 73.3689347099994,
"ci": [
73.3689347099994,
73.3689347099994
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "73.369"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "whoosh",
"value": 48.78160093999895,
"ci": [
48.78160093999895,
48.78160093999895
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "48.782"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "replace_ms_per_op",
"derived": true,
"warm": "hms",
"other": "sqlite-fts5",
"factor": 3.14805061888973,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "3.15x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms",
"value": 0.79275055,
"ci": [
0.79275055,
0.79275055
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0.793"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "hms-eager",
"value": 0.71397007,
"ci": [
0.71397007,
0.71397007
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0.714"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "sqlite-fts5",
"value": 6.279815929999586,
"ci": [
6.279815929999586,
6.279815929999586
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "6.280"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "lucene",
"value": 1.26921317,
"ci": [
1.26921317,
1.26921317
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1.269"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "tantivy",
"value": 63.29283183000144,
"ci": [
63.29283183000144,
63.29283183000144
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "63.293"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "whoosh",
"value": 6.582569769998372,
"ci": [
6.582569769998372,
6.582569769998372
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "6.583"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"system": "bm25s",
"value": null,
"ci": null,
"n": 0,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "n/m"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "delete_ms_per_op",
"derived": true,
"warm": "hms",
"other": "lucene",
"factor": 1.601024647665019,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "1.60x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "hms-eager",
"value": 2.3764114379882812,
"ci": [
2.3764114379882812,
2.3764114379882812
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2.4"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "sqlite-fts5",
"value": 0.0,
"ci": [
0.0,
0.0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "0.0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "lucene",
"value": 5.715169906616211,
"ci": [
5.715169906616211,
5.715169906616211
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "5.7"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "tantivy",
"value": 13.801004409790039,
"ci": [
13.801004409790039,
13.801004409790039
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "13.8"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "whoosh",
"value": 1.9728384017944336,
"ci": [
1.9728384017944336,
1.9728384017944336
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "2.0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"system": "bm25s",
"value": 56.54034996032715,
"ci": [
56.54034996032715,
56.54034996032715
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "deterministic",
"text": "56.5"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_mib",
"derived": true,
"warm": "hms",
"other": "whoosh",
"factor": 0.8301754360619107,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.83x"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "hms-eager",
"value": 92,
"ci": [
92,
92
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "92"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "sqlite-fts5",
"value": 0,
"ci": [
0,
0
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "0"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "lucene",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "tantivy",
"value": 1,
"ci": [
1,
1
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "1"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "whoosh",
"value": 2,
"ci": [
2,
2
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "2"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"system": "bm25s",
"value": 3,
"ci": [
3,
3
],
"n": 1,
"source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.install",
"validity": "valid",
"validity_reason": "",
"uncertainty": "unreplicated",
"text": "3"
},
{
"section": "beir:scifact (5,183 documents, 809 queries)",
"file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json",
"hardware_class": "modal-x86_64-8vcpu-32g",
"metric": "install_distributions",
"derived": true,
"warm": "hms",
"other": "tantivy",
"factor": 0.010869565217391304,
"factor_infinite": false,
"ci": null,
"verdict": "n/v",
"text": "0.01x"
}
]