issundb-core 0.1.0-alpha.2

IssunDB's storage engine and core data structures
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use std::path::Path;

use byteorder::BE;
use heed::{
    Database, DatabaseFlags, Env, EnvOpenOptions,
    types::{Bytes, Str, U64, Unit},
};

use crate::error::Error;

/// All LMDB sub-databases for IssunDB.
///
/// `out_adj` and `in_adj` use `DUPSORT + DUPFIXED`: each duplicate value is
/// one raw `AdjEntry` (20 bytes). A single `put` adds one entry in O(log n);
/// no read-modify-write of a blob is needed.
///
/// `label_idx` and `type_idx` use composite keys `(u32 BE, u64 BE)` = 12 bytes
/// for prefix-range scans by label or edge type.
pub struct Storage {
    pub env: Env,

    // Core records
    pub nodes: Database<U64<BE>, Bytes>, // node_id → msgpack NodeRecord
    pub edges: Database<U64<BE>, Bytes>, // edge_id → msgpack EdgeRecord

    // Adjacency: DUPSORT+DUPFIXED, one AdjEntry (20 B) per duplicate value
    pub out_adj: Database<U64<BE>, Bytes>, // node_id → [AdjEntry...]
    pub in_adj: Database<U64<BE>, Bytes>,  // node_id → [AdjEntry...]

    // Secondary indexes: composite key (u32 BE, u64 BE) → ()
    pub label_idx: Database<Bytes, Unit>, // (LabelId, NodeId) → ()
    pub type_idx: Database<Bytes, Unit>,  // (TypeId, EdgeId) → ()

    // Property indexes
    pub node_prop_idx: Database<Bytes, Unit>,
    pub edge_prop_idx: Database<Bytes, Unit>,

    // Full-text search databases
    pub fts_postings: Database<Bytes, Bytes>, // composite key (LabelId, PropKeyId, term) → DUPSORT [NodeId BE, frequency BE]
    pub fts_docs: Database<Bytes, Bytes>,     // (LabelId, PropKeyId, NodeId) → doc_len u32 BE

    // Vector embeddings (usearch HNSW, added later)
    pub vectors: Database<U64<BE>, Bytes>, // node_id → raw f32 bytes

    // Metadata + counters
    pub meta: Database<Str, Bytes>, // string key → bytes
}

impl Storage {
    pub fn open(path: &Path, map_size_gb: usize) -> Result<Self, Error> {
        std::fs::create_dir_all(path)?;

        let env = unsafe {
            EnvOpenOptions::new()
                .map_size(map_size_gb * 1024 * 1024 * 1024)
                .max_dbs(12)
                .open(path)?
        };

        let mut wtxn = env.write_txn()?;

        let nodes = env.create_database(&mut wtxn, Some("nodes"))?;
        let edges = env.create_database(&mut wtxn, Some("edges"))?;

        let out_adj = env
            .database_options()
            .types::<U64<BE>, Bytes>()
            .name("out_adj")
            .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
            .create(&mut wtxn)?;

        let in_adj = env
            .database_options()
            .types::<U64<BE>, Bytes>()
            .name("in_adj")
            .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
            .create(&mut wtxn)?;

        let label_idx = env.create_database(&mut wtxn, Some("label_idx"))?;
        let type_idx = env.create_database(&mut wtxn, Some("type_idx"))?;
        let node_prop_idx = env.create_database(&mut wtxn, Some("node_prop_idx"))?;
        let edge_prop_idx = env.create_database(&mut wtxn, Some("edge_prop_idx"))?;

        let fts_postings = env
            .database_options()
            .types::<Bytes, Bytes>()
            .name("fts_postings")
            .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
            .create(&mut wtxn)?;

        let fts_docs = env.create_database(&mut wtxn, Some("fts_docs"))?;

        let vectors = env.create_database(&mut wtxn, Some("vectors"))?;
        let meta = env.create_database(&mut wtxn, Some("meta"))?;

        wtxn.commit()?;

        Ok(Self {
            env,
            nodes,
            edges,
            out_adj,
            in_adj,
            label_idx,
            type_idx,
            node_prop_idx,
            edge_prop_idx,
            fts_postings,
            fts_docs,
            vectors,
            meta,
        })
    }
}