hypersteeldb 0.5.4

A database that compiles questions instead of guessing answers: typed vocabulary discovered from your documents, queries type-checked before they run, roaring-bitmap set algebra over reified hyperedges, and Dempster-Shafer evidence with an explicit conflict guard.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! Auto-profiler: sniff the constrained bitmap and route to the bitmap programs that carry the most
//! information — deterministically, no model in the loop. Needle only supplies `constraints`; this picks
//! the programs. For each facet we run a `breakdown` over the anchor (cheap roaring intersections) and
//! score it by `support × entropy` — a facet that is both well-populated and actually varies is
//! informative; a near-constant or ultra-sparse facet is not. We then surface: the top facets'
//! distributions, a `crosstab` of the two best low-cardinality facets (that's where "for each A, which B"
//! falls out), `rank` for a high-cardinality entity facet, and `s_clusters` when structure is dense.

use crate::db::Corpus;
use serde_json::{json, Value};

/// Facets that are structural noise for profiling (source tags, ambient rel/src).
fn skip_facet(f: &str) -> bool {
    matches!(f, "src" | "rel" | "source")
}

fn shannon(counts: &[u64], total: u64) -> f64 {
    if total == 0 {
        return 0.0;
    }
    -counts
        .iter()
        .filter(|&&c| c > 0)
        .map(|&c| {
            let p = c as f64 / total as f64;
            p * p.ln()
        })
        .sum::<f64>()
}

struct FacetStat {
    facet: String,
    cardinality: usize,
    support: u64,
    score: f64,
    breakdown: Value,
}

/// Sniff every facet over `anchor` and score by informativeness (support × (entropy+ε)); low/degenerate
/// cardinality is down-weighted.
fn sniff(corpus: &Corpus, anchor: &str) -> Vec<FacetStat> {
    let mut out = Vec::new();
    for f in corpus.facet_names() {
        if skip_facet(&f) {
            continue;
        }
        let bd = corpus.breakdown(anchor, &f, 50);
        let counts: Vec<u64> = bd
            .get("partition")
            .and_then(|p| p.as_array())
            .map(|a| a.iter().filter_map(|x| x.get("count").and_then(|c| c.as_u64())).collect())
            .unwrap_or_default();
        let support: u64 = counts.iter().sum();
        let card = counts.len();
        if support == 0 || card == 0 {
            continue;
        }
        let entropy = shannon(&counts, support);
        // Balanced categorical (2..=25 values) is most informative; huge-cardinality (entities) still
        // useful but down-weighted (better shown via rank); single-value facets carry no contrast.
        let card_w = match card {
            1 => 0.0,
            2..=25 => 1.0,
            _ => 0.35,
        };
        let score = support as f64 * (entropy + 0.1) * card_w;
        out.push(FacetStat { facet: f, cardinality: card, support, score, breakdown: bd });
    }
    out.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
    out
}

/// Build a composite `profile` result: the auto-routed program outputs over the constrained set. Each
/// section carries a real program payload so `crate::agent::synth` renders it deterministically.
pub fn auto_profile(corpus: &Corpus, anchor: &str) -> Value {
    let stats = sniff(corpus, anchor);
    if stats.is_empty() {
        return json!({ "program": "profile", "anchor": anchor, "sections": [], "note": "no informative facets in scope" });
    }
    let total = stats.iter().map(|s| s.support).max().unwrap_or(0);
    let mut sections: Vec<Value> = Vec::new();

    // 1) Distributions of the top informative facets.
    for s in stats.iter().take(3) {
        sections.push(json!({ "kind": "breakdown", "score": s.score, "result": s.breakdown }));
    }

    // 2) Crosstab of the two best low-cardinality categorical facets (2..=15) — the relational view.
    let low: Vec<&FacetStat> = stats.iter().filter(|s| (2..=15).contains(&s.cardinality)).collect();
    if low.len() >= 2 {
        let ct = corpus.crosstab(anchor, &low[0].facet, &low[1].facet, 8);
        sections.push(json!({ "kind": "crosstab", "result": ct }));
    }

    // 3) Rank a high-cardinality entity-like facet (top salient values).
    if let Some(hi) = stats.iter().find(|s| s.cardinality > 15) {
        sections.push(json!({ "kind": "rank", "result": corpus.rank(&hi.facet, 6) }));
    }

    // 4) Structural clusters when the co-occurrence graph is non-trivial.
    let clusters = corpus.s_clusters(2, 4);
    if clusters.get("clusters").and_then(|c| c.as_array()).map(|a| !a.is_empty()).unwrap_or(false) {
        sections.push(json!({ "kind": "clusters", "result": clusters }));
    }

    json!({ "program": "profile", "anchor": anchor, "scope_size": total, "sections": sections })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn shannon_entropy_basics() {
        assert!((shannon(&[5, 5], 10) - std::f64::consts::LN_2).abs() < 1e-9); // balanced 2-way = ln2
        assert!(shannon(&[10], 10).abs() < 1e-9); // single value = 0
        assert!(shannon(&[9, 1], 10) < shannon(&[5, 5], 10)); // skewed < balanced
    }
}