hypersteeldb 0.5.5

A database that compiles questions instead of guessing answers: typed vocabulary discovered from your documents, queries type-checked before they run, roaring-bitmap set algebra over reified hyperedges, and Dempster-Shafer evidence with an explicit conflict guard.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! Proves the multilingual (EN/JA/KO) model2vec embedder + Sinkhorn ontology sensing run natively in
//! Rust against the real shipped artifact (`step0_bundle/model2vec/potion.f32` + tokenizer.json).
//!
//! Requires the `embed` feature: model2vec links `tokenizers`, which the roaring core deliberately does
//! not depend on so that step 0 and the linter can target wasm32.
#![cfg(feature = "embed")]

use std::path::PathBuf;
use steeldb::text::ot;
use steeldb::text::{cosine, Model2Vec};

/// model2vec dir (potion.f32 + tokenizer.json). Set `STEELDB_TEST_MODEL2VEC` to run these tests;
/// otherwise they skip.
fn bundle() -> PathBuf {
    std::env::var("STEELDB_TEST_MODEL2VEC").map(PathBuf::from).unwrap_or_default()
}

#[test]
fn multilingual_embedding_runs_in_rust() {
    let dir = bundle();
    if !dir.join("potion.f32").exists() {
        eprintln!("skip: model2vec artifact not present");
        return;
    }
    let m = Model2Vec::load(&dir).expect("load potion.f32 + tokenizer");
    eprintln!("model2vec: {} rows × {} dim", m.rows(), m.dim());

    // one phrase per script; each must tokenize and embed
    let cases = [
        ("en", "the vehicle emission standard for diesel engines"),
        ("ja", "ディーゼルエンジンの排出ガス規制の基準"),
        ("ko", "디젤 엔진의 배출가스 규제 기준"),
    ];
    let mut vecs = Vec::new();
    for (lang, text) in cases {
        let tc = m.token_count(text);
        let v = m.embed(text).unwrap_or_else(|| panic!("{lang}: no in-vocab tokens"));
        assert_eq!(v.len(), m.dim());
        let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
        assert!((norm - 1.0).abs() < 1e-3, "{lang}: not L2-normalised ({norm})");
        eprintln!("{lang}: {tc} tokens, |v|={norm:.4}");
        vecs.push(v);
    }

    // identical text → cosine 1.0
    let a = m.embed("emission standard").unwrap();
    let b = m.embed("emission standard").unwrap();
    assert!((cosine(&a, &b) - 1.0).abs() < 1e-4);
}

#[test]
fn sinkhorn_codebook_forms() {
    let dir = bundle();
    if !dir.join("potion.f32").exists() {
        return;
    }
    let m = Model2Vec::load(&dir).unwrap();
    let terms = [
        "diesel engine", "gasoline engine", "electric motor",
        "brake system", "brake pad", "brake fluid",
        "emission limit", "exhaust gas", "particulate matter",
        "tire pressure", "wheel alignment", "suspension travel",
    ];
    let x: Vec<Vec<f32>> = terms.iter().filter_map(|t| m.embed(t)).collect();
    assert!(x.len() >= 8);
    let (protos, assign, cost) = ot::codebook(&x, 4, 0.05);
    assert!(protos.len() >= 2 && protos.len() <= 4);
    assert_eq!(assign.len(), x.len());
    assert!(assign.iter().all(|&a| a < protos.len()));
    assert!(cost.is_finite());
    eprintln!("codebook: {} facets, transport cost {cost:.4}", protos.len());
}