skanda_engine 0.1.0

A zero-dependency, ultra-high-performance retrieval engine designed for the next generation of RAG.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

pub fn encode_varint(mut value: u32) -> ([u8; 5], usize) {
    let mut buffer = [0u8; 5];
    let mut i = 0;
    while value >= 0x80 {
        buffer[i] = (value as u8 & 0x7F) | 0x80;
        value >>= 7;
        i += 1;
    }
    buffer[i] = (value as u8) & 0x7F;
    (buffer, i + 1)
}

pub fn decode_varint(bytes: &[u8], cursor: &mut usize) -> u32 {
    let mut value = 0u32;
    let mut shift = 0;
    while *cursor < bytes.len() {
        let byte = bytes[*cursor];
        *cursor += 1;
        value |= ((byte & 0x7F) as u32) << shift;
        if byte & 0x80 == 0 {
            return value;
        }
        shift += 7;
    }
    value
}

pub fn encode_delta(ids: &[u32]) -> Vec<u8> {
    if ids.is_empty() { return vec![]; }
    let mut encoded = Vec::new();
    let mut last = 0;
    for &id in ids {
        let delta = id - last;
        let (buf, len) = encode_varint(delta);
        encoded.extend_from_slice(&buf[..len]);
        last = id;
    }
    encoded
}

pub fn decode_delta(bytes: &[u8], count: usize) -> Vec<u32> {
    let mut ids = Vec::with_capacity(count);
    let mut last = 0;
    let mut cursor = 0;
    for _ in 0..count {
        if cursor >= bytes.len() { break; }
        let delta = decode_varint(bytes, &mut cursor);
        let id = last + delta;
        ids.push(id);
        last = id;
    }
    ids
}

/// Encodes a mapping of BlockID -> [BytePositions]
pub fn encode_inverted_entry(block_positions: &[(u32, Vec<u32>)]) -> Vec<u8> {
    let mut encoded = Vec::new();
    let mut last_block_id = 0;
    
    for (block_id, positions) in block_positions {
        // 1. Encode BlockID Delta
        let block_delta = block_id - last_block_id;
        let (buf, len) = encode_varint(block_delta);
        encoded.extend_from_slice(&buf[..len]);
        last_block_id = *block_id;
        
        // 2. Encode Position Count
        let (buf, len) = encode_varint(positions.len() as u32);
        encoded.extend_from_slice(&buf[..len]);
        
        // 3. Encode Position Deltas
        let mut last_pos = 0;
        for &pos in positions {
            let pos_delta = pos - last_pos;
            let (buf, len) = encode_varint(pos_delta);
            encoded.extend_from_slice(&buf[..len]);
            last_pos = pos;
        }
    }
    encoded
}

pub fn decode_inverted_entry(bytes: &[u8]) -> Vec<(u32, Vec<u32>)> {
    let mut results = Vec::new();
    let mut cursor = 0;
    let mut last_block_id = 0;
    
    while cursor < bytes.len() {
        // 1. Decode BlockID
        let block_delta = decode_varint(bytes, &mut cursor);
        let block_id = last_block_id + block_delta;
        last_block_id = block_id;
        
        // 2. Decode Position Count
        let pos_count = decode_varint(bytes, &mut cursor) as usize;
        
        // 3. Decode Positions
        let mut positions = Vec::with_capacity(pos_count);
        let mut last_pos = 0;
        for _ in 0..pos_count {
            let pos_delta = decode_varint(bytes, &mut cursor);
            let pos = last_pos + pos_delta;
            positions.push(pos);
            last_pos = pos;
        }
        
        results.push((block_id, positions));
    }
    results
}