hypersteeldb 0.5.3

A database that compiles questions instead of guessing answers: typed vocabulary discovered from your documents, queries type-checked before they run, roaring-bitmap set algebra over reified hyperedges, and Dempster-Shafer evidence with an explicit conflict guard.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! Bench: `HashSet<u32>` (the TS `Set<number>` engine) vs roaring, on real infon postings.
//!
//! `Usage: bench <situations.jsonl> [max_lines]`
//! Builds both indexes from one load, then for a suite of IKL queries verifies the two backends
//! return identical result cardinality (parity) and times each. Reports build time and posting
//! memory too, since the roaring win is both speed and footprint.

use std::path::Path;
use std::time::Instant;
use steeldb::index::InfonIndex;
use steeldb::tokenql::evaluate;
use steeldb::{Postings, RoarPostings, SetPostings};

const QUERIES: &[(&str, &str)] = &[
    ("dense ∩ dense", "(and source/sentinel-2-l2a class/vegetation)"),
    ("dense − sparse", "(and class/vegetation (not class/water))"),
    ("or of years", "(or time/2023 time/2024)"),
    ("sparse ∩ dense", "(and time/2024 class/water)"),
    ("3-way and", "(and source/sentinel-2-l2a time/2024 class/vegetation)"),
    ("not (closed-world)", "(not source/sentinel-2-l2a)"),
    ("wildcard union", "rel/part_whole/within/+^/*"),
    ("wildcard ∩ class", "(and rel/part_whole/within/+^/* class/vegetation)"),
];

fn time_query<B: Postings, S: steeldb::TokenStore<B>>(store: &S, expr: &str, iters: u32) -> (usize, f64) {
    // warm
    let card = evaluate(store, expr).len();
    let t = Instant::now();
    for _ in 0..iters {
        std::hint::black_box(evaluate(store, expr).len());
    }
    let us = t.elapsed().as_secs_f64() * 1e6 / iters as f64;
    (card, us)
}

fn human_bytes(b: usize) -> String {
    let b = b as f64;
    if b >= 1e9 {
        format!("{:.2} GB", b / 1e9)
    } else if b >= 1e6 {
        format!("{:.1} MB", b / 1e6)
    } else if b >= 1e3 {
        format!("{:.1} KB", b / 1e3)
    } else {
        format!("{b} B")
    }
}

fn main() {
    let args: Vec<String> = std::env::args().collect();
    let path = args.get(1).map(|s| s.as_str()).unwrap_or_else(|| {
        eprintln!("usage: bench <situations.jsonl> [max_lines]");
        std::process::exit(2);
    });
    let max_lines = args.get(2).and_then(|s| s.parse::<usize>().ok());

    let t = Instant::now();
    let (raw, n) = steeldb::jsonl::load(Path::new(path), max_lines).expect("load jsonl");
    let load_s = t.elapsed().as_secs_f64();
    let n_postings: usize = raw.values().map(|v| v.len()).sum();
    println!(
        "loaded {n} situations, {} tokens, {n_postings} postings in {load_s:.2}s\n",
        raw.len()
    );

    let t = Instant::now();
    let set_ix: InfonIndex<SetPostings> = InfonIndex::from_postings(raw.clone(), n);
    let set_build = t.elapsed().as_secs_f64();

    let t = Instant::now();
    let roar_ix: InfonIndex<RoarPostings> = InfonIndex::from_postings(raw, n);
    let roar_build = t.elapsed().as_secs_f64();

    println!("{:<24} {:>14} {:>14}", "", "HashSet<u32>", "roaring");
    println!(
        "{:<24} {:>14} {:>14}",
        "build",
        format!("{set_build:.2}s"),
        format!("{roar_build:.2}s")
    );
    println!(
        "{:<24} {:>14} {:>14}",
        "postings in memory",
        human_bytes(set_ix.postings_native_bytes()),
        human_bytes(roar_ix.postings_native_bytes())
    );
    println!(
        "{:<24} {:>14} {:>14}  (portable, identical)",
        "postings delta-gap",
        human_bytes(set_ix.postings_deltagap_bytes()),
        human_bytes(roar_ix.postings_deltagap_bytes())
    );

    println!(
        "\n{:<22} {:>10} {:>13} {:>13} {:>8}",
        "query", "result", "HashSet µs", "roaring µs", "speedup"
    );
    println!("{}", "-".repeat(70));
    let iters = 200u32;
    for (label, expr) in QUERIES {
        let (c1, set_us) = time_query(&set_ix, expr, iters);
        let (c2, roar_us) = time_query(&roar_ix, expr, iters);
        let parity = if c1 == c2 { "" } else { "  ‼ MISMATCH" };
        println!(
            "{:<22} {:>10} {:>13.1} {:>13.1} {:>7.1}x{}",
            label,
            c1,
            set_us,
            roar_us,
            set_us / roar_us,
            parity
        );
        assert_eq!(c1, c2, "parity failure on `{expr}`: {c1} vs {c2}");
    }
    println!("\nparity: all queries returned identical cardinality on both backends.");
}