array-format 0.10.0

A block-backed, footer-indexed container for storing multiple n-dimensional arrays in a single file, with a delta/overlay architecture and pluggable compression and storage backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

//! Block identifiers and chunk address types.
//!
//! A [`ChunkAddress`] locates a slice of array data within a specific
//! block by `(block_id, offset, size)`.

use rkyv::{Archive, Deserialize, Serialize};

/// A block-relative allocation: which block, and the byte range within it.
///
/// The write-side counterpart of [`ChunkAddress`], using `u64` offsets/sizes
/// while a layer is being built; it narrows to [`ChunkAddress`] on flush.
#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, Archive, Serialize, Deserialize)]
pub(crate) struct BlockAllocAddress {
    id: BlockId,
    offset: u64,
    size: u64,
}

/// A block identifier within the file.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Archive, Serialize, Deserialize)]
pub struct BlockId(pub u32);

/// Locates a contiguous slice of bytes within a block.
///
/// To read the data: find the block identified by [`block_id`](ChunkAddress::block_id),
/// seek to [`offset`](ChunkAddress::offset) bytes into the decompressed block,
/// and read [`size`](ChunkAddress::size) bytes.
#[derive(Debug, Clone, PartialEq, Archive, Serialize, Deserialize)]
pub struct ChunkAddress {
    /// Which block the data resides in.
    pub block_id: BlockId,
    /// Byte offset within the decompressed block.
    pub offset: u32,
    /// Number of bytes to read from the block.
    pub size: u32,
}

impl BlockAllocAddress {
    /// Creates an address for `size` bytes at `offset` within block `id`.
    pub(crate) fn new(id: BlockId, offset: u64, size: u64) -> Self {
        Self { id, offset, size }
    }

    /// The block this allocation lives in.
    pub(crate) fn id(&self) -> BlockId {
        self.id
    }

    /// Byte offset of the allocation within the block.
    pub(crate) fn offset(&self) -> u64 {
        self.offset
    }

    /// Length of the allocation in bytes.
    pub(crate) fn size(&self) -> u64 {
        self.size
    }
}

impl From<BlockAllocAddress> for ChunkAddress {
    fn from(a: BlockAllocAddress) -> Self {
        ChunkAddress {
            block_id: a.id,
            offset: a.offset as u32,
            size: a.size as u32,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn block_id_copy() {
        let a = BlockId(7);
        let b = a;
        assert_eq!(a, b);
    }

    #[test]
    fn chunk_address_fields() {
        let addr = ChunkAddress {
            block_id: BlockId(3),
            offset: 1024,
            size: 4096,
        };
        assert_eq!(addr.block_id, BlockId(3));
        assert_eq!(addr.offset, 1024);
        assert_eq!(addr.size, 4096);
    }
}