murr 0.2.0

Columnar in-memory cache for AI/ML inference workloads
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

use std::sync::Arc;

use arrow::{
    array::{ArrayRef, RecordBatch},
    datatypes::{DataType, Field, Schema},
};

use crate::{
    core::MurrError,
    io::{
        column::{ColumnEncoder, encoder_for},
        schema::{SegmentColumnSchema, SegmentSchema},
    },
};

pub struct ReadRow<'a> {
    pub schema: &'a SegmentSchema,
    pub bitset: &'a [u8],
    pub values: &'a [u8],
}

impl<'a> ReadRow<'a> {
    pub fn new(schema: &'a SegmentSchema, raw: &'a [u8]) -> Self {
        let (bitset, values) = raw.split_at(schema.bitset_size);
        Self {
            schema,
            bitset,
            values,
        }
    }

    pub fn is_null(&self, column: &SegmentColumnSchema) -> bool {
        let idx = column.index as usize;
        let byte = idx / 8;
        let bit = (idx % 8) as u8;
        (self.bitset[byte] >> bit) & 1 == 1
    }

    pub fn read_static<T: bytemuck::Pod>(&self, column: &SegmentColumnSchema) -> T {
        let start = column.offset as usize;
        let end = start + std::mem::size_of::<T>();
        bytemuck::pod_read_unaligned(&self.values[start..end])
    }

    pub fn read_dynamic(&self, column: &SegmentColumnSchema) -> &[u8] {
        let slot = column.offset as usize;
        let payload_off =
            u32::from_le_bytes(self.values[slot..slot + 4].try_into().unwrap()) as usize;
        let len = u32::from_le_bytes(
            self.values[payload_off..payload_off + 4]
                .try_into()
                .unwrap(),
        ) as usize;
        &self.values[payload_off + 4..payload_off + 4 + len]
    }
}

/// Accumulates rows into Arrow column builders inside `Store::read`. Stores
/// fan raw bytes through `add_row` / `add_empty`; the builder yields the final
/// `RecordBatch` via `build`. Keeps slice lifetimes bounded by the store fn
/// frame so backends like LMDB can hold a read txn across the iteration.
pub struct ReadBatchBuilder<'a> {
    segment: &'a SegmentSchema,
    columns: Vec<&'a SegmentColumnSchema>,
    encoders: Vec<Box<dyn ColumnEncoder>>,
}

impl<'a> ReadBatchBuilder<'a> {
    pub fn new(
        segment: &'a SegmentSchema,
        columns: Vec<&'a SegmentColumnSchema>,
        capacity: usize,
    ) -> Self {
        let encoders = columns.iter().map(|c| encoder_for(c, capacity)).collect();
        Self {
            segment,
            columns,
            encoders,
        }
    }

    pub fn add_row(&mut self, bytes: &[u8]) -> Result<(), MurrError> {
        let row = ReadRow::new(self.segment, bytes);
        for e in &mut self.encoders {
            e.add_row(&row)?;
        }
        Ok(())
    }

    pub fn add_empty(&mut self) -> Result<(), MurrError> {
        for e in &mut self.encoders {
            e.add_empty()?;
        }
        Ok(())
    }

    pub fn build(mut self) -> Result<RecordBatch, MurrError> {
        let arrays: Vec<ArrayRef> = self.encoders.iter_mut().map(|e| e.build()).collect();
        let fields: Vec<Field> = self
            .columns
            .iter()
            .map(|c| Field::new(&c.name, DataType::from(&c.dtype), true))
            .collect();
        RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)
            .map_err(|e| MurrError::ArrowError(e.to_string()))
    }
}