hypersteeldb 0.5.2

A database that compiles questions instead of guessing answers: typed vocabulary discovered from your documents, queries type-checked before they run, roaring-bitmap set algebra over reified hyperedges, and Dempster-Shafer evidence with an explicit conflict guard.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

//! Streaming JSONL loader. Each line is one situation `{ "tokens": [...], "num": {...} }`; the line
//! index (0-based) is the situation id. Builds `token -> ascending sids` in a single pass — the sids
//! land in id order for free, so the index build hits the fast sorted path.

use serde::Deserialize;
use std::collections::HashMap;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;

#[derive(Deserialize)]
struct Row {
    #[serde(default)]
    tokens: Vec<String>,
}

/// Returns (token -> sids, n_situations).
pub fn load(path: &Path, max_lines: Option<usize>) -> std::io::Result<(HashMap<String, Vec<u32>>, u32)> {
    let file = File::open(path)?;
    let reader = BufReader::with_capacity(1 << 20, file);
    let mut by_token: HashMap<String, Vec<u32>> = HashMap::new();
    let mut sid: u32 = 0;

    for line in reader.lines() {
        let line = line?;
        if line.trim().is_empty() {
            continue;
        }
        if let Some(max) = max_lines {
            if sid as usize >= max {
                break;
            }
        }
        let row: Row = match serde_json::from_str(&line) {
            Ok(r) => r,
            Err(_) => {
                sid += 1;
                continue;
            }
        };
        for tok in row.tokens {
            by_token.entry(tok).or_default().push(sid);
        }
        sid += 1;
    }
    Ok((by_token, sid))
}