harmos-storage 0.1.7

Harmos storage: durable journals, snapshots and Arrow-backed stream storage.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

/* ***
 *
 * Harmos
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * ***
 */

// Storage Arrow Query Contract

/* === Modules === */

/* === Re-Exports === */

/* === Imports === */

// Standard Library Imports
use std::collections::BTreeMap;
// Third Party Imports
use arrow_array::RecordBatch;
use arrow_data::ArrayData;
// Local Imports
use super::invalid;
use crate::Result;

/* === Constants === */

/* === Macros === */

/* === Type Aliases === */

/* === Structs === */

/* === Enums === */

/* === Traits === */

/* === Implementations === */

/* === Public Functions === */

pub(super) fn resident_bytes(batch: &RecordBatch) -> Result<usize> {
    let mut seen = BTreeMap::new();
    let mut bytes = super::schema_memory(batch.schema().as_ref());
    for column in batch.columns() {
        count(&column.to_data(), &mut seen, &mut bytes)?;
    }
    Ok(bytes)
}

/* === Private Functions === */

fn count(data: &ArrayData, seen: &mut BTreeMap<usize, usize>, bytes: &mut usize) -> Result<()> {
    *bytes = bytes.checked_add(std::mem::size_of::<ArrayData>()).ok_or_else(|| invalid("Arrow resident size overflow"))?;
    for buffer in data.buffers().iter().chain(data.nulls().map(|nulls| nulls.buffer())) {
        let base =
            (buffer.as_ptr() as usize).checked_sub(buffer.ptr_offset()).ok_or_else(|| invalid("Arrow allocation offset underflow"))?;
        // Include Accessible Spans When Custom Buffers Report Zero Capacity.
        let span = buffer.len().checked_add(buffer.ptr_offset()).ok_or_else(|| invalid("Arrow accessible allocation span overflow"))?;
        let capacity = buffer.capacity().max(span);
        let previous = seen.entry(base).or_insert(0);
        if capacity > *previous {
            *bytes = bytes.checked_add(capacity - *previous).ok_or_else(|| invalid("Arrow resident size overflow"))?;
            *previous = capacity;
        }
    }
    for child in data.child_data() {
        count(child, seen, bytes)?;
    }
    Ok(())
}

/* === Tests === */